Merge pull request #3380 from nextcloud/issue-3341-voice-message-loop

Fixes not able to execute audio messages

app/src/main/java/com/nextcloud/talk/chat/ChatActivity.kt

@@ -193,7 +193,7 @@ import com.nextcloud.talk.ui.dialog.ShowReactionsDialog
 import com.nextcloud.talk.ui.recyclerview.MessageSwipeActions
 import com.nextcloud.talk.ui.recyclerview.MessageSwipeCallback
 import com.nextcloud.talk.utils.ApiUtils
-import com.nextcloud.talk.utils.AudioUtils.audioFileToFloatArray
+import com.nextcloud.talk.utils.AudioUtils
 import com.nextcloud.talk.utils.ContactUtils
 import com.nextcloud.talk.utils.ConversationUtils
 import com.nextcloud.talk.utils.DateConstants
@@ -507,6 +507,7 @@ class ChatActivity :
             val text = getString(roomToken, "")
             binding.messageInputView.messageInput.setText(text)
         }
+        this.lifecycle.addObserver(AudioUtils)
     }
 
     override fun onStop() {
@@ -530,6 +531,7 @@ class ChatActivity :
                 apply()
             }
         }
+        this.lifecycle.removeObserver(AudioUtils)
     }
 
     @Suppress("LongMethod")
@@ -909,7 +911,7 @@ class ChatActivity :
             message.isDownloadingVoiceMessage = true
             adapter?.update(message)
             CoroutineScope(Dispatchers.Default).launch {
-                val r = audioFileToFloatArray(file)
+                val r = AudioUtils.audioFileToFloatArray(file)
                 message.voiceMessageFloatArray = r
                 withContext(Dispatchers.Main) {
                     startPlayback(message)

app/src/main/java/com/nextcloud/talk/utils/AudioUtils.kt

@@ -27,6 +27,8 @@ import android.media.MediaExtractor
 import android.media.MediaFormat
 import android.os.SystemClock
 import android.util.Log
+import androidx.lifecycle.DefaultLifecycleObserver
+import androidx.lifecycle.LifecycleOwner
 import java.io.File
 import java.io.IOException
 import java.nio.ByteOrder
@@ -38,11 +40,26 @@ import kotlin.math.abs
  * AudioUtils are for processing raw audio using android's low level APIs, for more information read here
  * [MediaCodec documentation](https://developer.android.com/reference/android/media/MediaCodec)
  */
-object AudioUtils {
+object AudioUtils : DefaultLifecycleObserver {
     private val TAG = AudioUtils::class.java.simpleName
     private const val VALUE_10 = 10
     private const val TIME_LIMIT = 5000
     private const val DEFAULT_SIZE = 500
+    private enum class LifeCycleFlag {
+        PAUSED,
+        RESUMED
+    }
+    private lateinit var currentLifeCycleFlag: LifeCycleFlag
+
+    override fun onResume(owner: LifecycleOwner) {
+        super.onResume(owner)
+        currentLifeCycleFlag = LifeCycleFlag.RESUMED
+    }
+
+    override fun onPause(owner: LifecycleOwner) {
+        super.onPause(owner)
+        currentLifeCycleFlag = LifeCycleFlag.PAUSED
+    }
 
     /**
      * Suspension function, returns a FloatArray of size 500, containing the values of an audio file squeezed between
@@ -51,25 +68,55 @@ object AudioUtils {
     @Throws(IOException::class)
     suspend fun audioFileToFloatArray(file: File): FloatArray {
         return suspendCoroutine {
+            // Used to keep track of the time it took to process the audio file
             val startTime = SystemClock.elapsedRealtime()
-            var result = mutableListOf<Float>()
+
+            // Always a FloatArray of Size 500
+            var result: MutableList<Float>? = mutableListOf()
+
+            // Setting the file path to the audio file
             val path = file.path
             val mediaExtractor = MediaExtractor()
             mediaExtractor.setDataSource(path)
 
+            // Basically just boilerplate to set up meta data for the audio file
             val mediaFormat = mediaExtractor.getTrackFormat(0)
+            // Frame rate is required for encoders, optional for decoders. So we set it to null here.
             mediaFormat.setString(MediaFormat.KEY_FRAME_RATE, null)
             mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 0)
 
             mediaExtractor.release()
 
+            // More Boiler plate to set up the codec
             val mediaCodecList = MediaCodecList(MediaCodecList.ALL_CODECS)
             val codecName = mediaCodecList.findDecoderForFormat(mediaFormat)
             val mediaCodec = MediaCodec.createByCodecName(codecName)
+
+            /**
+             ************************************ Media Codec *******************************************
+             *                                        │
+             *                      INPUT BUFFERS     │            OUTPUT BUFFERS
+             *                                        │
+             * ┌────────────────┐             ┌───────┴────────┐              ┌─────────────────┐
+             * │                │ Empty Buffer│                │ Filled Buffer│                 │
+             * │                │    [][][]   │                │ [-][-][-]    │                 │
+             * │                │ ◄───────────┤                ├────────────► │                 │
+             * │     Client     │             │     Codec      │              │      Client     │
+             * │                │             │                │              │                 │
+             * │                ├───────────► │                │ ◄────────────┤                 │
+             * │                │ [-][-][-]   │                │   [][][]     │                 │
+             * └────────────────┘Filled Buffer└───────┬────────┘Empty Buffer  └─────────────────┘
+             *                                        │
+             *   Client provides                      │                         Client consumes
+             *   input Data                           │                         output data
+             *
+             ********************************************************************************************
+             */
             mediaCodec.setCallback(object : MediaCodec.Callback() {
                 private var extractor: MediaExtractor? = null
                 val tempList = mutableListOf<Float>()
                 override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
+                    // Setting up the extractor if not already done
                     if (extractor == null) {
                         extractor = MediaExtractor()
                         try {
@@ -79,6 +126,8 @@ object AudioUtils {
                             e.printStackTrace()
                         }
                     }
+
+                    // Boiler plate, Extracts a buffer of encoded audio data to be sent to the codec for processing
                     val byteBuffer = codec.getInputBuffer(index)
                     if (byteBuffer != null) {
                         val sampleSize = extractor!!.readSampleData(byteBuffer, 0)
@@ -96,6 +145,7 @@ object AudioUtils {
                 }
 
                 override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
+                    // Boiler plate to get the audio data in a usable form
                     val outputBuffer = codec.getOutputBuffer(index)
                     val bufferFormat = codec.getOutputFormat(index)
                     val samples = outputBuffer!!.order(ByteOrder.nativeOrder()).asShortBuffer()
@@ -104,23 +154,37 @@ object AudioUtils {
                         return
                     }
                     val sampleLength = (samples.remaining() / numChannels)
+
                     // Squeezes the value of each sample between [0,1) using y = (x-1)/x
                     for (i in 0 until sampleLength) {
                         val x = abs(samples[i * numChannels + index].toInt()) / VALUE_10
                         val y = (if (x > 0) ((x - 1) / x.toFloat()) else x.toFloat())
                         tempList.add(y)
                     }
+
                     codec.releaseOutputBuffer(index, false)
+
+                    // Cancels the process if it ends, exceeds the time limit, or the activity falls out of view
                     val currTime = SystemClock.elapsedRealtime() - startTime
-                    if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 || currTime > TIME_LIMIT) {
+                    if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 ||
+                        currTime > TIME_LIMIT ||
+                        currentLifeCycleFlag == LifeCycleFlag.PAUSED
+                    ) {
+                        Log.d(
+                            TAG,
+                            "Processing ended with time: $currTime \n" +
+                                "Is finished: ${info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0} \n" +
+                                "Lifecycle state: $currentLifeCycleFlag"
+                        )
                         codec.stop()
                         codec.release()
                         extractor!!.release()
                         extractor = null
-                        if (currTime < TIME_LIMIT) {
+                        result = if (currTime < TIME_LIMIT) {
-                            result = tempList
+                            tempList
                         } else {
-                            Log.d(TAG, "time limit exceeded")
+                            Log.e(TAG, "Error in MediaCodec Callback:\n\tonOutputBufferAvailable: Time limit exceeded")
+                            null
                         }
                     }
                 }
@@ -131,20 +195,30 @@ object AudioUtils {
                     codec.release()
                     extractor!!.release()
                     extractor = null
-                    result = tempList
+                    result = null
                 }
 
                 override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
                     // unused atm
                 }
             })
+
+            // More Boiler plate to start the codec
             mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
             mediaCodec.configure(mediaFormat, null, null, 0)
             mediaCodec.start()
-            while (result.size <= 0) {
+
+            // This runs until the codec finishes or the time limit is exceeded, or an error occurs
+            // If the time limit is exceed or an error occurs, the result should be null
+            while (result != null && result!!.size <= 0) {
                 continue
             }
-            it.resume(shrinkFloatArray(result.toFloatArray(), DEFAULT_SIZE))
+
+            if (result != null && result!!.size > DEFAULT_SIZE) {
+                it.resume(shrinkFloatArray(result!!.toFloatArray(), DEFAULT_SIZE))
+            } else {
+                it.resume(FloatArray(DEFAULT_SIZE))
+            }
         }
     }