From 79b366c5ba5122cf9b468d804b518a410d3a2dac Mon Sep 17 00:00:00 2001 From: Julius Linus Date: Thu, 12 Oct 2023 10:55:47 -0500 Subject: [PATCH] Refactoring AudioUtils - Fixes a bug with the processing taking to long - Added a bunch of comments, to make maintaining this more easier - Added LifeCycle Awareness Signed-off-by: Julius Linus --- .../com/nextcloud/talk/chat/ChatActivity.kt | 6 +- .../com/nextcloud/talk/utils/AudioUtils.kt | 92 +++++++++++++++++-- 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt b/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt index cc0c7bd9f..887d44233 100644 --- a/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt @@ -193,7 +193,7 @@ import com.nextcloud.talk.ui.dialog.ShowReactionsDialog import com.nextcloud.talk.ui.recyclerview.MessageSwipeActions import com.nextcloud.talk.ui.recyclerview.MessageSwipeCallback import com.nextcloud.talk.utils.ApiUtils -import com.nextcloud.talk.utils.AudioUtils.audioFileToFloatArray +import com.nextcloud.talk.utils.AudioUtils import com.nextcloud.talk.utils.ContactUtils import com.nextcloud.talk.utils.ConversationUtils import com.nextcloud.talk.utils.DateConstants @@ -507,6 +507,7 @@ class ChatActivity : val text = getString(roomToken, "") binding.messageInputView.messageInput.setText(text) } + this.lifecycle.addObserver(AudioUtils) } override fun onStop() { @@ -530,6 +531,7 @@ class ChatActivity : apply() } } + this.lifecycle.removeObserver(AudioUtils) } @Suppress("LongMethod") @@ -909,7 +911,7 @@ class ChatActivity : message.isDownloadingVoiceMessage = true adapter?.update(message) CoroutineScope(Dispatchers.Default).launch { - val r = audioFileToFloatArray(file) + val r = AudioUtils.audioFileToFloatArray(file) message.voiceMessageFloatArray = r withContext(Dispatchers.Main) { startPlayback(message) diff --git a/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt b/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt index c64d58724..8151ad39e 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt +++ b/app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt @@ -27,6 +27,8 @@ import android.media.MediaExtractor import android.media.MediaFormat import android.os.SystemClock import android.util.Log +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner import java.io.File import java.io.IOException import java.nio.ByteOrder @@ -38,11 +40,26 @@ import kotlin.math.abs * AudioUtils are for processing raw audio using android's low level APIs, for more information read here * [MediaCodec documentation](https://developer.android.com/reference/android/media/MediaCodec) */ -object AudioUtils { +object AudioUtils : DefaultLifecycleObserver { private val TAG = AudioUtils::class.java.simpleName private const val VALUE_10 = 10 private const val TIME_LIMIT = 5000 private const val DEFAULT_SIZE = 500 + private enum class LifeCycleFlag { + PAUSED, + RESUMED + } + private lateinit var currentLifeCycleFlag: LifeCycleFlag + + override fun onResume(owner: LifecycleOwner) { + super.onResume(owner) + currentLifeCycleFlag = LifeCycleFlag.RESUMED + } + + override fun onPause(owner: LifecycleOwner) { + super.onPause(owner) + currentLifeCycleFlag = LifeCycleFlag.PAUSED + } /** * Suspension function, returns a FloatArray of size 500, containing the values of an audio file squeezed between @@ -51,25 +68,55 @@ object AudioUtils { @Throws(IOException::class) suspend fun audioFileToFloatArray(file: File): FloatArray { return suspendCoroutine { + // Used to keep track of the time it took to process the audio file val startTime = SystemClock.elapsedRealtime() - var result = mutableListOf() + + // Always a FloatArray of Size 500 + var result: MutableList? = mutableListOf() + + // Setting the file path to the audio file val path = file.path val mediaExtractor = MediaExtractor() mediaExtractor.setDataSource(path) + // Basically just boilerplate to set up meta data for the audio file val mediaFormat = mediaExtractor.getTrackFormat(0) + // Frame rate is required for encoders, optional for decoders. So we set it to null here. mediaFormat.setString(MediaFormat.KEY_FRAME_RATE, null) mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 0) mediaExtractor.release() + // More Boiler plate to set up the codec val mediaCodecList = MediaCodecList(MediaCodecList.ALL_CODECS) val codecName = mediaCodecList.findDecoderForFormat(mediaFormat) val mediaCodec = MediaCodec.createByCodecName(codecName) + + /** + ************************************ Media Codec ******************************************* + * │ + * INPUT BUFFERS │ OUTPUT BUFFERS + * │ + * ┌────────────────┐ ┌───────┴────────┐ ┌─────────────────┐ + * │ │ Empty Buffer│ │ Filled Buffer│ │ + * │ │ [][][] │ │ [-][-][-] │ │ + * │ │ ◄───────────┤ ├────────────► │ │ + * │ Client │ │ Codec │ │ Client │ + * │ │ │ │ │ │ + * │ ├───────────► │ │ ◄────────────┤ │ + * │ │ [-][-][-] │ │ [][][] │ │ + * └────────────────┘Filled Buffer└───────┬────────┘Empty Buffer └─────────────────┘ + * │ + * Client provides │ Client consumes + * input Data │ output data + * + ******************************************************************************************** + */ mediaCodec.setCallback(object : MediaCodec.Callback() { private var extractor: MediaExtractor? = null val tempList = mutableListOf() override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { + // Setting up the extractor if not already done if (extractor == null) { extractor = MediaExtractor() try { @@ -79,6 +126,8 @@ object AudioUtils { e.printStackTrace() } } + + // Boiler plate, Extracts a buffer of encoded audio data to be sent to the codec for processing val byteBuffer = codec.getInputBuffer(index) if (byteBuffer != null) { val sampleSize = extractor!!.readSampleData(byteBuffer, 0) @@ -96,6 +145,7 @@ object AudioUtils { } override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) { + // Boiler plate to get the audio data in a usable form val outputBuffer = codec.getOutputBuffer(index) val bufferFormat = codec.getOutputFormat(index) val samples = outputBuffer!!.order(ByteOrder.nativeOrder()).asShortBuffer() @@ -104,23 +154,37 @@ object AudioUtils { return } val sampleLength = (samples.remaining() / numChannels) + // Squeezes the value of each sample between [0,1) using y = (x-1)/x for (i in 0 until sampleLength) { val x = abs(samples[i * numChannels + index].toInt()) / VALUE_10 val y = (if (x > 0) ((x - 1) / x.toFloat()) else x.toFloat()) tempList.add(y) } + codec.releaseOutputBuffer(index, false) + + // Cancels the process if it ends, exceeds the time limit, or the activity falls out of view val currTime = SystemClock.elapsedRealtime() - startTime - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 || currTime > TIME_LIMIT) { + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 || + currTime > TIME_LIMIT || + currentLifeCycleFlag == LifeCycleFlag.PAUSED + ) { + Log.d( + TAG, + "Processing ended with time: $currTime \n" + + "Is finished: ${info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0} \n" + + "Lifecycle state: $currentLifeCycleFlag" + ) codec.stop() codec.release() extractor!!.release() extractor = null - if (currTime < TIME_LIMIT) { - result = tempList + result = if (currTime < TIME_LIMIT) { + tempList } else { - Log.d(TAG, "time limit exceeded") + Log.e(TAG, "Error in MediaCodec Callback:\n\tonOutputBufferAvailable: Time limit exceeded") + null } } } @@ -131,20 +195,30 @@ object AudioUtils { codec.release() extractor!!.release() extractor = null - result = tempList + result = null } override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { // unused atm } }) + + // More Boiler plate to start the codec mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) mediaCodec.configure(mediaFormat, null, null, 0) mediaCodec.start() - while (result.size <= 0) { + + // This runs until the codec finishes or the time limit is exceeded, or an error occurs + // If the time limit is exceed or an error occurs, the result should be null + while (result != null && result!!.size <= 0) { continue } - it.resume(shrinkFloatArray(result.toFloatArray(), DEFAULT_SIZE)) + + if (result != null && result!!.size > DEFAULT_SIZE) { + it.resume(shrinkFloatArray(result!!.toFloatArray(), DEFAULT_SIZE)) + } else { + it.resume(FloatArray(DEFAULT_SIZE)) + } } }