Browse Source
* Create `mediaupload` module for media pre-processing. * Split `mediapicker` and `mediaupload` modules.feature/jme/open-room-member-details-when-clicking-on-user-data
Jorge Martin Espinosa
1 year ago
committed by
GitHub
33 changed files with 1148 additions and 156 deletions
@ -1,6 +1,6 @@
@@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project version="4"> |
||||
<component name="KotlinJpsPluginSettings"> |
||||
<option name="version" value="1.8.20" /> |
||||
<option name="version" value="1.8.21" /> |
||||
</component> |
||||
</project> |
||||
@ -0,0 +1 @@
@@ -0,0 +1 @@
|
||||
Add media pre-processing before uploading it. |
||||
@ -0,0 +1,24 @@
@@ -0,0 +1,24 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.androidutils.media |
||||
|
||||
import android.media.MediaMetadataRetriever |
||||
|
||||
/** [MediaMetadataRetriever] only implements `AutoClosable` since API 29, so we need to execute this to have the same in older APIs. */ |
||||
inline fun <T> MediaMetadataRetriever.runAndRelease(block: MediaMetadataRetriever.() -> T): T { |
||||
return block().also { release() } |
||||
} |
||||
@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediapickers.api |
||||
|
||||
import android.net.Uri |
||||
import androidx.activity.result.PickVisualMediaRequest |
||||
import androidx.compose.runtime.Composable |
||||
|
||||
interface PickerProvider { |
||||
|
||||
@Composable |
||||
fun registerGalleryPicker( |
||||
onResult: (uri: Uri?, mimeType: String?) -> Unit |
||||
): PickerLauncher<PickVisualMediaRequest, Uri?> |
||||
|
||||
@Composable |
||||
fun registerFilePicker( |
||||
mimeType: String, |
||||
onResult: (Uri?) -> Unit |
||||
): PickerLauncher<String, Uri?> |
||||
|
||||
@Composable |
||||
fun registerCameraPhotoPicker(onResult: (Uri?) -> Unit): PickerLauncher<Uri, Boolean> |
||||
|
||||
@Composable |
||||
fun registerCameraVideoPicker(onResult: (Uri?) -> Unit): PickerLauncher<Uri, Boolean> |
||||
|
||||
} |
||||
@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
plugins { |
||||
id("io.element.android-compose-library") |
||||
alias(libs.plugins.anvil) |
||||
} |
||||
|
||||
android { |
||||
namespace = "io.element.android.libraries.mediapickers.impl" |
||||
|
||||
dependencies { |
||||
implementation(projects.libraries.core) |
||||
implementation(projects.libraries.di) |
||||
implementation(libs.inject) |
||||
api(projects.libraries.mediapickers.api) |
||||
} |
||||
} |
||||
@ -0,0 +1,31 @@
@@ -0,0 +1,31 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
plugins { |
||||
id("io.element.android-compose-library") |
||||
alias(libs.plugins.anvil) |
||||
} |
||||
|
||||
android { |
||||
namespace = "io.element.android.libraries.mediapickers.test" |
||||
|
||||
dependencies { |
||||
implementation(projects.libraries.core) |
||||
implementation(projects.libraries.di) |
||||
implementation(libs.inject) |
||||
api(projects.libraries.mediapickers.api) |
||||
} |
||||
} |
||||
@ -0,0 +1,58 @@
@@ -0,0 +1,58 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediapickers.test |
||||
|
||||
import android.net.Uri |
||||
import androidx.activity.result.PickVisualMediaRequest |
||||
import androidx.compose.runtime.Composable |
||||
import io.element.android.libraries.core.mimetype.MimeTypes |
||||
import io.element.android.libraries.mediapickers.api.NoOpPickerLauncher |
||||
import io.element.android.libraries.mediapickers.api.PickerLauncher |
||||
import io.element.android.libraries.mediapickers.api.PickerProvider |
||||
|
||||
class FakePickerProvider : PickerProvider { |
||||
private var mimeType = MimeTypes.Any |
||||
private var result: Uri? = null |
||||
|
||||
@Composable |
||||
override fun registerGalleryPicker(onResult: (uri: Uri?, mimeType: String?) -> Unit): PickerLauncher<PickVisualMediaRequest, Uri?> { |
||||
return NoOpPickerLauncher { onResult(result, mimeType) } |
||||
} |
||||
|
||||
@Composable |
||||
override fun registerFilePicker(mimeType: String, onResult: (Uri?) -> Unit): PickerLauncher<String, Uri?> { |
||||
return NoOpPickerLauncher { onResult(result) } |
||||
} |
||||
|
||||
@Composable |
||||
override fun registerCameraPhotoPicker(onResult: (Uri?) -> Unit): PickerLauncher<Uri, Boolean> { |
||||
return NoOpPickerLauncher { onResult(result) } |
||||
} |
||||
|
||||
@Composable |
||||
override fun registerCameraVideoPicker(onResult: (Uri?) -> Unit): PickerLauncher<Uri, Boolean> { |
||||
return NoOpPickerLauncher { onResult(result) } |
||||
} |
||||
|
||||
fun givenResult(value: Uri?) { |
||||
this.result = value |
||||
} |
||||
|
||||
fun givenMimeType(mimeType: String) { |
||||
this.mimeType = mimeType |
||||
} |
||||
} |
||||
@ -0,0 +1,42 @@
@@ -0,0 +1,42 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
plugins { |
||||
id("io.element.android-library") |
||||
alias(libs.plugins.anvil) |
||||
alias(libs.plugins.ksp) |
||||
} |
||||
|
||||
android { |
||||
namespace = "io.element.android.libraries.mediaupload.api" |
||||
|
||||
anvil { |
||||
generateDaggerFactories.set(true) |
||||
} |
||||
|
||||
dependencies { |
||||
implementation(projects.anvilannotations) |
||||
anvil(projects.anvilcodegen) |
||||
|
||||
implementation(projects.libraries.architecture) |
||||
implementation(projects.libraries.androidutils) |
||||
implementation(projects.libraries.core) |
||||
implementation(projects.libraries.di) |
||||
api(projects.libraries.matrix.api) |
||||
implementation(libs.inject) |
||||
implementation(libs.coroutines.core) |
||||
} |
||||
} |
||||
@ -0,0 +1,32 @@
@@ -0,0 +1,32 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload.api |
||||
|
||||
import android.net.Uri |
||||
|
||||
interface MediaPreProcessor { |
||||
/** |
||||
* Given a [uri] and [mediaType], pre-processes the media before it's uploaded, resizing, transcoding, and removing sensitive info from its metadata. |
||||
* If [deleteOriginal] is `true`, the file reference by the [uri] will be automatically deleted too when this process finishes. |
||||
* @return a [Result] with the [MediaUploadInfo] containing all the info needed to begin the upload. |
||||
*/ |
||||
suspend fun process( |
||||
uri: Uri, |
||||
mediaType: MediaType, |
||||
deleteOriginal: Boolean = false |
||||
): Result<MediaUploadInfo> |
||||
} |
||||
@ -0,0 +1,24 @@
@@ -0,0 +1,24 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload.api |
||||
|
||||
sealed interface MediaType { |
||||
object Image : MediaType |
||||
object Video : MediaType |
||||
object Audio : MediaType |
||||
object File : MediaType |
||||
} |
||||
@ -0,0 +1,36 @@
@@ -0,0 +1,36 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload.api |
||||
|
||||
import io.element.android.libraries.matrix.api.media.AudioInfo |
||||
import io.element.android.libraries.matrix.api.media.FileInfo |
||||
import io.element.android.libraries.matrix.api.media.ImageInfo |
||||
import io.element.android.libraries.matrix.api.media.ThumbnailInfo |
||||
import io.element.android.libraries.matrix.api.media.VideoInfo |
||||
import java.io.File |
||||
|
||||
sealed interface MediaUploadInfo { |
||||
data class Image(val file: File, val info: ImageInfo, val thumbnailInfo: ThumbnailProcessingInfo?) : MediaUploadInfo |
||||
data class Video(val file: File, val info: VideoInfo, val thumbnailInfo: ThumbnailProcessingInfo?) : MediaUploadInfo |
||||
data class Audio(val file: File, val info: AudioInfo) : MediaUploadInfo |
||||
data class AnyFile(val file: File, val info: FileInfo) : MediaUploadInfo |
||||
} |
||||
|
||||
data class ThumbnailProcessingInfo( |
||||
val file: File, |
||||
val info: ThumbnailInfo, |
||||
) |
||||
@ -0,0 +1,49 @@
@@ -0,0 +1,49 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
plugins { |
||||
id("io.element.android-library") |
||||
alias(libs.plugins.anvil) |
||||
alias(libs.plugins.ksp) |
||||
} |
||||
|
||||
android { |
||||
namespace = "io.element.android.libraries.mediaupload.impl" |
||||
|
||||
anvil { |
||||
generateDaggerFactories.set(true) |
||||
} |
||||
|
||||
dependencies { |
||||
implementation(projects.anvilannotations) |
||||
anvil(projects.anvilcodegen) |
||||
|
||||
api(projects.libraries.mediaupload.api) |
||||
implementation(projects.libraries.architecture) |
||||
implementation(projects.libraries.androidutils) |
||||
implementation(projects.libraries.core) |
||||
implementation(projects.libraries.di) |
||||
implementation(projects.libraries.matrix.api) |
||||
implementation(libs.inject) |
||||
implementation(libs.androidx.exifinterface) |
||||
implementation(libs.coroutines.core) |
||||
implementation(libs.otaliastudios.transcoder) |
||||
|
||||
testImplementation(libs.test.junit) |
||||
testImplementation(libs.coroutines.test) |
||||
testImplementation(libs.test.truth) |
||||
} |
||||
} |
||||
@ -0,0 +1,117 @@
@@ -0,0 +1,117 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload |
||||
|
||||
import android.content.Context |
||||
import android.graphics.Bitmap |
||||
import android.graphics.BitmapFactory |
||||
import io.element.android.libraries.androidutils.bitmap.calculateInSampleSize |
||||
import io.element.android.libraries.androidutils.bitmap.resizeToMax |
||||
import io.element.android.libraries.androidutils.bitmap.rotateToMetadataOrientation |
||||
import io.element.android.libraries.androidutils.file.createTmpFile |
||||
import io.element.android.libraries.di.ApplicationContext |
||||
import kotlinx.coroutines.Dispatchers |
||||
import kotlinx.coroutines.withContext |
||||
import java.io.BufferedInputStream |
||||
import java.io.File |
||||
import java.io.InputStream |
||||
import javax.inject.Inject |
||||
|
||||
class ImageCompressor @Inject constructor( |
||||
@ApplicationContext private val context: Context, |
||||
) { |
||||
|
||||
/** |
||||
* Decodes the [inputStream] into a [Bitmap] and applies the needed transformations (rotation, scale) based on [resizeMode], then writes it into a |
||||
* temporary file using the passed [format] and [desiredQuality]. |
||||
* @return a [Result] containing the resulting [ImageCompressionResult] with the temporary [File] and some metadata. |
||||
*/ |
||||
suspend fun compressToTmpFile( |
||||
inputStream: InputStream, |
||||
resizeMode: ResizeMode, |
||||
format: Bitmap.CompressFormat = Bitmap.CompressFormat.JPEG, |
||||
desiredQuality: Int = 80, |
||||
): Result<ImageCompressionResult> = withContext(Dispatchers.IO) { |
||||
runCatching { |
||||
val compressedBitmap = compressToBitmap(inputStream, resizeMode).getOrThrow() |
||||
|
||||
// Encode bitmap to the destination temporary file |
||||
val tmpFile = context.createTmpFile() |
||||
tmpFile.outputStream().use { |
||||
compressedBitmap.compress(format, desiredQuality, it) |
||||
} |
||||
|
||||
ImageCompressionResult(tmpFile, compressedBitmap.width, compressedBitmap.height, tmpFile.length()) |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Decodes the [inputStream] into a [Bitmap] and applies the needed transformations (rotation, scale) based on [resizeMode]. |
||||
* @return a [Result] containing the resulting [Bitmap]. |
||||
*/ |
||||
fun compressToBitmap( |
||||
inputStream: InputStream, |
||||
resizeMode: ResizeMode, |
||||
): Result<Bitmap> = runCatching { |
||||
BufferedInputStream(inputStream).use { input -> |
||||
val options = BitmapFactory.Options() |
||||
calculateDecodingScale(input, resizeMode, options) |
||||
val decodedBitmap = BitmapFactory.decodeStream(input, null, options) |
||||
?: error("Decoding Bitmap from InputStream failed") |
||||
val rotatedBitmap = decodedBitmap.rotateToMetadataOrientation(input).getOrThrow() |
||||
if (resizeMode is ResizeMode.Strict) { |
||||
rotatedBitmap.resizeToMax(resizeMode.maxWidth, resizeMode.maxHeight) |
||||
} else { |
||||
rotatedBitmap |
||||
} |
||||
} |
||||
} |
||||
|
||||
private fun calculateDecodingScale( |
||||
inputStream: BufferedInputStream, |
||||
resizeMode: ResizeMode, |
||||
options: BitmapFactory.Options |
||||
) { |
||||
val (width, height) = when (resizeMode) { |
||||
is ResizeMode.Approximate -> resizeMode.desiredWidth to resizeMode.desiredHeight |
||||
is ResizeMode.Strict -> (resizeMode.maxWidth / 2) to (resizeMode.maxHeight / 2) |
||||
is ResizeMode.None -> return |
||||
} |
||||
// Read bounds only |
||||
inputStream.mark(inputStream.available()) |
||||
options.inJustDecodeBounds = true |
||||
BitmapFactory.decodeStream(inputStream, null, options) |
||||
// Set sample size based on the outWidth and outHeight |
||||
options.inSampleSize = options.calculateInSampleSize(width, height) |
||||
// Now read the actual image and rotate it to match its metadata |
||||
inputStream.reset() |
||||
options.inJustDecodeBounds = false |
||||
} |
||||
} |
||||
|
||||
data class ImageCompressionResult( |
||||
val file: File, |
||||
val width: Int, |
||||
val height: Int, |
||||
val size: Long, |
||||
) |
||||
|
||||
sealed interface ResizeMode { |
||||
object None : ResizeMode |
||||
data class Approximate(val desiredWidth: Int, val desiredHeight: Int) : ResizeMode |
||||
data class Strict(val maxWidth: Int, val maxHeight: Int) : ResizeMode |
||||
} |
||||
@ -0,0 +1,263 @@
@@ -0,0 +1,263 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload |
||||
|
||||
import android.content.Context |
||||
import android.graphics.Bitmap |
||||
import android.media.MediaMetadataRetriever |
||||
import android.net.Uri |
||||
import androidx.exifinterface.media.ExifInterface |
||||
import com.squareup.anvil.annotations.ContributesBinding |
||||
import io.element.android.libraries.androidutils.file.createTmpFile |
||||
import io.element.android.libraries.androidutils.media.runAndRelease |
||||
import io.element.android.libraries.core.data.tryOrNull |
||||
import io.element.android.libraries.core.mimetype.MimeTypes |
||||
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeAudio |
||||
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeImage |
||||
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeVideo |
||||
import io.element.android.libraries.di.AppScope |
||||
import io.element.android.libraries.di.ApplicationContext |
||||
import io.element.android.libraries.matrix.api.media.AudioInfo |
||||
import io.element.android.libraries.matrix.api.media.FileInfo |
||||
import io.element.android.libraries.matrix.api.media.ImageInfo |
||||
import io.element.android.libraries.matrix.api.media.ThumbnailInfo |
||||
import io.element.android.libraries.matrix.api.media.VideoInfo |
||||
import io.element.android.libraries.mediaupload.api.MediaPreProcessor |
||||
import io.element.android.libraries.mediaupload.api.MediaType |
||||
import io.element.android.libraries.mediaupload.api.MediaUploadInfo |
||||
import io.element.android.libraries.mediaupload.api.ThumbnailProcessingInfo |
||||
import kotlinx.coroutines.Dispatchers |
||||
import kotlinx.coroutines.flow.catch |
||||
import kotlinx.coroutines.flow.filterIsInstance |
||||
import kotlinx.coroutines.flow.first |
||||
import kotlinx.coroutines.flow.onEach |
||||
import kotlinx.coroutines.withContext |
||||
import java.io.ByteArrayInputStream |
||||
import java.io.ByteArrayOutputStream |
||||
import java.io.File |
||||
import java.io.InputStream |
||||
import javax.inject.Inject |
||||
import kotlin.time.Duration.Companion.seconds |
||||
|
||||
@ContributesBinding(AppScope::class) |
||||
class MediaPreProcessorImpl @Inject constructor( |
||||
@ApplicationContext private val context: Context, |
||||
private val imageCompressor: ImageCompressor, |
||||
private val videoCompressor: VideoCompressor, |
||||
) : MediaPreProcessor { |
||||
companion object { |
||||
/** |
||||
* Used for calculating `inSampleSize` for bitmaps. |
||||
* |
||||
* *Note*: Ideally, this should result in images of up to (but not included) 1280x1280 being sent. However, images with very different width and height |
||||
* values may surpass this limit. (i.e.: an image of `480x3000px` would have `inSampleSize=1` and be sent as is). |
||||
*/ |
||||
private const val IMAGE_SCALE_REF_SIZE = 640 |
||||
|
||||
/** |
||||
* Max width of thumbnail images. |
||||
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails). |
||||
*/ |
||||
private const val THUMB_MAX_WIDTH = 800 |
||||
/** |
||||
* Max height of thumbnail images. |
||||
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails). |
||||
*/ |
||||
private const val THUMB_MAX_HEIGHT = 600 |
||||
|
||||
/** |
||||
* Frame of the video to be used for generating a thumbnail. |
||||
*/ |
||||
private val VIDEO_THUMB_FRAME = 5.seconds.inWholeMicroseconds |
||||
} |
||||
|
||||
private val contentResolver = context.contentResolver |
||||
|
||||
override suspend fun process( |
||||
uri: Uri, |
||||
mediaType: MediaType, |
||||
deleteOriginal: Boolean, |
||||
): Result<MediaUploadInfo> = runCatching { |
||||
// Camera returns an 'octet-stream' mimetype, so it needs to be overridden |
||||
val originalMimeType = contentResolver.getType(uri) |
||||
val mimeType = when (mediaType) { |
||||
MediaType.Image -> MimeTypes.Images |
||||
MediaType.Video -> MimeTypes.Videos |
||||
MediaType.Audio -> MimeTypes.Audio |
||||
else -> originalMimeType |
||||
} |
||||
val compressBeforeSending = mediaType in sequenceOf(MediaType.Image, MediaType.Video) |
||||
val result = if (compressBeforeSending && mimeType != MimeTypes.Gif) { |
||||
when { |
||||
mimeType.isMimeTypeImage() -> processImage(uri) |
||||
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType) |
||||
mimeType.isMimeTypeAudio() -> processAudio(uri) |
||||
else -> error("Cannot compress file of type: $mimeType") |
||||
} |
||||
} else { |
||||
val file = copyToTmpFile(uri) |
||||
// Remove image metadata here too |
||||
if (mimeType.isMimeTypeImage() && mimeType != MimeTypes.Gif) { |
||||
removeSensitiveImageMetadata(file) |
||||
} |
||||
val info = FileInfo( |
||||
mimetype = originalMimeType, |
||||
size = file.length(), |
||||
thumbnailInfo = null, |
||||
thumbnailUrl = null, |
||||
) |
||||
MediaUploadInfo.AnyFile(file, info) |
||||
} |
||||
|
||||
if (deleteOriginal) { |
||||
contentResolver.delete(uri, null, null) |
||||
} |
||||
|
||||
result |
||||
} |
||||
|
||||
private suspend fun processImage(uri: Uri): MediaUploadInfo { |
||||
val compressedFileResult = contentResolver.openInputStream(uri).use { input -> |
||||
imageCompressor.compressToTmpFile( |
||||
inputStream = requireNotNull(input), |
||||
resizeMode = ResizeMode.Approximate(IMAGE_SCALE_REF_SIZE, IMAGE_SCALE_REF_SIZE), |
||||
).getOrThrow() |
||||
} |
||||
|
||||
removeSensitiveImageMetadata(compressedFileResult.file) |
||||
|
||||
val thumbnailResult = compressedFileResult.file.inputStream().use { generateImageThumbnail(it) } |
||||
val processingResult = compressedFileResult.toImageInfo(MimeTypes.Jpeg, thumbnailResult?.file?.path, thumbnailResult?.info) |
||||
return MediaUploadInfo.Image(compressedFileResult.file, processingResult, thumbnailResult) |
||||
} |
||||
|
||||
private suspend fun processVideo(uri: Uri, mimeType: String?): MediaUploadInfo { |
||||
val thumbnailInfo = extractVideoThumbnail(uri) |
||||
val resultFile = videoCompressor.compress(uri) |
||||
.onEach { |
||||
// TODO handle progress |
||||
} |
||||
.filterIsInstance<VideoTranscodingEvent.Completed>() |
||||
.first() |
||||
.file |
||||
|
||||
val videoProcessingInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo?.file?.path, thumbnailInfo?.info) |
||||
return MediaUploadInfo.Video(resultFile, videoProcessingInfo, thumbnailInfo) |
||||
} |
||||
|
||||
private suspend fun processAudio(uri: Uri): MediaUploadInfo { |
||||
val file = copyToTmpFile(uri) |
||||
return MediaMetadataRetriever().runAndRelease { |
||||
setDataSource(context, Uri.fromFile(file)) |
||||
|
||||
val info = AudioInfo( |
||||
duration = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L, |
||||
size = file.length() |
||||
) |
||||
|
||||
MediaUploadInfo.Audio(file, info) |
||||
} |
||||
} |
||||
|
||||
private suspend fun generateImageThumbnail(inputStream: InputStream): ThumbnailProcessingInfo? { |
||||
val thumbnailResult = imageCompressor |
||||
.compressToTmpFile( |
||||
inputStream = inputStream, |
||||
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT), |
||||
).getOrNull() |
||||
|
||||
return thumbnailResult?.toThumbnailProcessingInfo(MimeTypes.Jpeg) |
||||
} |
||||
|
||||
private fun removeSensitiveImageMetadata(file: File) { |
||||
// Remove GPS info, user comments and subject location tags |
||||
val exifInterface = ExifInterface(file) |
||||
// See ExifInterface.TAG_GPS_INFO_IFD_POINTER |
||||
exifInterface.setAttribute("GPSInfoIFDPointer", null) |
||||
exifInterface.setAttribute(ExifInterface.TAG_USER_COMMENT, null) |
||||
exifInterface.setAttribute(ExifInterface.TAG_SUBJECT_LOCATION, null) |
||||
tryOrNull { exifInterface.saveAttributes() } |
||||
} |
||||
|
||||
private suspend fun createTmpFileWithInput(inputStream: InputStream): File? { |
||||
return withContext(Dispatchers.IO) { |
||||
tryOrNull { |
||||
val tmpFile = context.createTmpFile() |
||||
tmpFile.outputStream().use { inputStream.copyTo(it) } |
||||
tmpFile |
||||
} |
||||
} |
||||
} |
||||
|
||||
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailUrl: String?, thumbnailInfo: ThumbnailInfo?): VideoInfo = |
||||
MediaMetadataRetriever().runAndRelease { |
||||
setDataSource(context, Uri.fromFile(file)) |
||||
|
||||
VideoInfo( |
||||
duration = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L, |
||||
width = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toLong() ?: 0L, |
||||
height = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L, |
||||
mimetype = mimeType, |
||||
size = file.length(), |
||||
thumbnailInfo = thumbnailInfo, |
||||
thumbnailUrl = thumbnailUrl, |
||||
blurhash = null, |
||||
) |
||||
} |
||||
|
||||
private suspend fun extractVideoThumbnail(uri: Uri): ThumbnailProcessingInfo? = |
||||
MediaMetadataRetriever().runAndRelease { |
||||
setDataSource(context, uri) |
||||
val bitmap = getFrameAtTime(VIDEO_THUMB_FRAME) ?: return@runAndRelease null |
||||
val inputStream = ByteArrayOutputStream().use { |
||||
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, it) |
||||
ByteArrayInputStream(it.toByteArray()) |
||||
} |
||||
|
||||
val result = imageCompressor.compressToTmpFile( |
||||
inputStream = inputStream, |
||||
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT), |
||||
) |
||||
|
||||
result.getOrThrow().toThumbnailProcessingInfo(MimeTypes.Jpeg) |
||||
} |
||||
|
||||
private suspend fun copyToTmpFile(uri: Uri): File { |
||||
return contentResolver.openInputStream(uri)?.use { createTmpFileWithInput(it) } |
||||
?: error("Could not copy the contents of $uri to a temporary file") |
||||
} |
||||
} |
||||
|
||||
fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailUrl: String?, thumbnailInfo: ThumbnailInfo?) = ImageInfo( |
||||
width = width.toLong(), |
||||
height = height.toLong(), |
||||
mimetype = mimeType, |
||||
size = size, |
||||
thumbnailInfo = thumbnailInfo, |
||||
thumbnailUrl = thumbnailUrl, |
||||
blurhash = null, |
||||
) |
||||
|
||||
fun ImageCompressionResult.toThumbnailProcessingInfo(mimeType: String) = ThumbnailProcessingInfo( |
||||
file = file, |
||||
info = ThumbnailInfo( |
||||
width = width.toLong(), |
||||
height = height.toLong(), |
||||
mimetype = mimeType, |
||||
size = size, |
||||
), |
||||
) |
||||
@ -0,0 +1,71 @@
@@ -0,0 +1,71 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload |
||||
|
||||
import android.content.Context |
||||
import android.net.Uri |
||||
import com.otaliastudios.transcoder.Transcoder |
||||
import com.otaliastudios.transcoder.TranscoderListener |
||||
import io.element.android.libraries.androidutils.file.createTmpFile |
||||
import io.element.android.libraries.di.ApplicationContext |
||||
import kotlinx.coroutines.channels.awaitClose |
||||
import kotlinx.coroutines.flow.callbackFlow |
||||
import java.io.File |
||||
import javax.inject.Inject |
||||
|
||||
class VideoCompressor @Inject constructor( |
||||
@ApplicationContext private val context: Context, |
||||
) { |
||||
|
||||
fun compress(uri: Uri) = callbackFlow { |
||||
val tmpFile = context.createTmpFile() |
||||
val future = Transcoder.into(tmpFile.path) |
||||
.addDataSource(context, uri) |
||||
.setListener(object : TranscoderListener { |
||||
override fun onTranscodeProgress(progress: Double) { |
||||
trySend(VideoTranscodingEvent.Progress(progress.toFloat())) |
||||
} |
||||
|
||||
override fun onTranscodeCompleted(successCode: Int) { |
||||
trySend(VideoTranscodingEvent.Completed(tmpFile)) |
||||
close() |
||||
} |
||||
|
||||
override fun onTranscodeCanceled() { |
||||
tmpFile.delete() |
||||
close() |
||||
} |
||||
|
||||
override fun onTranscodeFailed(exception: Throwable) { |
||||
tmpFile.delete() |
||||
close(exception) |
||||
} |
||||
}) |
||||
.transcode() |
||||
|
||||
awaitClose { |
||||
if (!future.isDone) { |
||||
future.cancel(true) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
sealed interface VideoTranscodingEvent { |
||||
data class Progress(val value: Float) : VideoTranscodingEvent |
||||
data class Completed(val file: File) : VideoTranscodingEvent |
||||
} |
||||
@ -0,0 +1,27 @@
@@ -0,0 +1,27 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
plugins { |
||||
id("io.element.android-library") |
||||
} |
||||
|
||||
android { |
||||
namespace = "io.element.android.libraries.matrix.test" |
||||
} |
||||
|
||||
dependencies { |
||||
api(projects.libraries.mediaupload.api) |
||||
} |
||||
@ -0,0 +1,44 @@
@@ -0,0 +1,44 @@
|
||||
/* |
||||
* Copyright (c) 2023 New Vector Ltd |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package io.element.android.libraries.mediaupload.test |
||||
|
||||
import android.net.Uri |
||||
import io.element.android.libraries.matrix.api.media.FileInfo |
||||
import io.element.android.libraries.mediaupload.api.MediaPreProcessor |
||||
import io.element.android.libraries.mediaupload.api.MediaType |
||||
import io.element.android.libraries.mediaupload.api.MediaUploadInfo |
||||
import java.io.File |
||||
|
||||
class FakeMediaPreProcessor : MediaPreProcessor { |
||||
|
||||
private var result: Result<MediaUploadInfo> = Result.success( |
||||
MediaUploadInfo.AnyFile( |
||||
File("test"), |
||||
FileInfo( |
||||
mimetype = "*/*", |
||||
size = 999L, |
||||
thumbnailInfo = null, |
||||
thumbnailUrl = null, |
||||
) |
||||
) |
||||
) |
||||
override suspend fun process(uri: Uri, mediaType: MediaType, deleteOriginal: Boolean): Result<MediaUploadInfo> = result |
||||
|
||||
fun givenResult(value: Result<MediaUploadInfo>) { |
||||
this.result = value |
||||
} |
||||
} |
||||
Loading…
Reference in new issue