mirror of
https://github.com/nextcloud/talk-android
synced 2025-02-01 12:11:59 +00:00
Refactoring AudioUtils
- Fixes a bug with the processing taking to long - Added a bunch of comments, to make maintaining this more easier - Added LifeCycle Awareness Signed-off-by: Julius Linus <julius.linus@nextcloud.com>
This commit is contained in:
Julius Linus
Marcel Hibbe
parent
d3f55b0f33
commit
79b366c5ba
@ -193,7 +193,7 @@ import com.nextcloud.talk.ui.dialog.ShowReactionsDialog
|
||||
import com.nextcloud.talk.ui.recyclerview.MessageSwipeActions
|
||||
import com.nextcloud.talk.ui.recyclerview.MessageSwipeCallback
|
||||
import com.nextcloud.talk.utils.ApiUtils
|
||||
import com.nextcloud.talk.utils.AudioUtils.audioFileToFloatArray
|
||||
import com.nextcloud.talk.utils.AudioUtils
|
||||
import com.nextcloud.talk.utils.ContactUtils
|
||||
import com.nextcloud.talk.utils.ConversationUtils
|
||||
import com.nextcloud.talk.utils.DateConstants
|
||||
@ -507,6 +507,7 @@ class ChatActivity :
|
||||
val text = getString(roomToken, "")
|
||||
binding.messageInputView.messageInput.setText(text)
|
||||
}
|
||||
this.lifecycle.addObserver(AudioUtils)
|
||||
}
|
||||
|
||||
override fun onStop() {
|
||||
@ -530,6 +531,7 @@ class ChatActivity :
|
||||
apply()
|
||||
}
|
||||
}
|
||||
this.lifecycle.removeObserver(AudioUtils)
|
||||
}
|
||||
|
||||
@Suppress("LongMethod")
|
||||
@ -909,7 +911,7 @@ class ChatActivity :
|
||||
message.isDownloadingVoiceMessage = true
|
||||
adapter?.update(message)
|
||||
CoroutineScope(Dispatchers.Default).launch {
|
||||
val r = audioFileToFloatArray(file)
|
||||
val r = AudioUtils.audioFileToFloatArray(file)
|
||||
message.voiceMessageFloatArray = r
|
||||
withContext(Dispatchers.Main) {
|
||||
startPlayback(message)
|
||||
|
||||
@ -27,6 +27,8 @@ import android.media.MediaExtractor
|
||||
import android.media.MediaFormat
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import androidx.lifecycle.DefaultLifecycleObserver
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.nio.ByteOrder
|
||||
@ -38,11 +40,26 @@ import kotlin.math.abs
|
||||
* AudioUtils are for processing raw audio using android's low level APIs, for more information read here
|
||||
* [MediaCodec documentation](https://developer.android.com/reference/android/media/MediaCodec)
|
||||
*/
|
||||
object AudioUtils {
|
||||
object AudioUtils : DefaultLifecycleObserver {
|
||||
private val TAG = AudioUtils::class.java.simpleName
|
||||
private const val VALUE_10 = 10
|
||||
private const val TIME_LIMIT = 5000
|
||||
private const val DEFAULT_SIZE = 500
|
||||
private enum class LifeCycleFlag {
|
||||
PAUSED,
|
||||
RESUMED
|
||||
}
|
||||
private lateinit var currentLifeCycleFlag: LifeCycleFlag
|
||||
|
||||
override fun onResume(owner: LifecycleOwner) {
|
||||
super.onResume(owner)
|
||||
currentLifeCycleFlag = LifeCycleFlag.RESUMED
|
||||
}
|
||||
|
||||
override fun onPause(owner: LifecycleOwner) {
|
||||
super.onPause(owner)
|
||||
currentLifeCycleFlag = LifeCycleFlag.PAUSED
|
||||
}
|
||||
|
||||
/**
|
||||
* Suspension function, returns a FloatArray of size 500, containing the values of an audio file squeezed between
|
||||
@ -51,25 +68,55 @@ object AudioUtils {
|
||||
@Throws(IOException::class)
|
||||
suspend fun audioFileToFloatArray(file: File): FloatArray {
|
||||
return suspendCoroutine {
|
||||
// Used to keep track of the time it took to process the audio file
|
||||
val startTime = SystemClock.elapsedRealtime()
|
||||
var result = mutableListOf<Float>()
|
||||
|
||||
// Always a FloatArray of Size 500
|
||||
var result: MutableList<Float>? = mutableListOf()
|
||||
|
||||
// Setting the file path to the audio file
|
||||
val path = file.path
|
||||
val mediaExtractor = MediaExtractor()
|
||||
mediaExtractor.setDataSource(path)
|
||||
|
||||
// Basically just boilerplate to set up meta data for the audio file
|
||||
val mediaFormat = mediaExtractor.getTrackFormat(0)
|
||||
// Frame rate is required for encoders, optional for decoders. So we set it to null here.
|
||||
mediaFormat.setString(MediaFormat.KEY_FRAME_RATE, null)
|
||||
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 0)
|
||||
|
||||
mediaExtractor.release()
|
||||
|
||||
// More Boiler plate to set up the codec
|
||||
val mediaCodecList = MediaCodecList(MediaCodecList.ALL_CODECS)
|
||||
val codecName = mediaCodecList.findDecoderForFormat(mediaFormat)
|
||||
val mediaCodec = MediaCodec.createByCodecName(codecName)
|
||||
|
||||
/**
|
||||
************************************ Media Codec *******************************************
|
||||
* │
|
||||
* INPUT BUFFERS │ OUTPUT BUFFERS
|
||||
* │
|
||||
* ┌────────────────┐ ┌───────┴────────┐ ┌─────────────────┐
|
||||
* │ │ Empty Buffer│ │ Filled Buffer│ │
|
||||
* │ │ [][][] │ │ [-][-][-] │ │
|
||||
* │ │ ◄───────────┤ ├────────────► │ │
|
||||
* │ Client │ │ Codec │ │ Client │
|
||||
* │ │ │ │ │ │
|
||||
* │ ├───────────► │ │ ◄────────────┤ │
|
||||
* │ │ [-][-][-] │ │ [][][] │ │
|
||||
* └────────────────┘Filled Buffer└───────┬────────┘Empty Buffer └─────────────────┘
|
||||
* │
|
||||
* Client provides │ Client consumes
|
||||
* input Data │ output data
|
||||
*
|
||||
********************************************************************************************
|
||||
*/
|
||||
mediaCodec.setCallback(object : MediaCodec.Callback() {
|
||||
private var extractor: MediaExtractor? = null
|
||||
val tempList = mutableListOf<Float>()
|
||||
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
|
||||
// Setting up the extractor if not already done
|
||||
if (extractor == null) {
|
||||
extractor = MediaExtractor()
|
||||
try {
|
||||
@ -79,6 +126,8 @@ object AudioUtils {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
// Boiler plate, Extracts a buffer of encoded audio data to be sent to the codec for processing
|
||||
val byteBuffer = codec.getInputBuffer(index)
|
||||
if (byteBuffer != null) {
|
||||
val sampleSize = extractor!!.readSampleData(byteBuffer, 0)
|
||||
@ -96,6 +145,7 @@ object AudioUtils {
|
||||
}
|
||||
|
||||
override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
|
||||
// Boiler plate to get the audio data in a usable form
|
||||
val outputBuffer = codec.getOutputBuffer(index)
|
||||
val bufferFormat = codec.getOutputFormat(index)
|
||||
val samples = outputBuffer!!.order(ByteOrder.nativeOrder()).asShortBuffer()
|
||||
@ -104,23 +154,37 @@ object AudioUtils {
|
||||
return
|
||||
}
|
||||
val sampleLength = (samples.remaining() / numChannels)
|
||||
|
||||
// Squeezes the value of each sample between [0,1) using y = (x-1)/x
|
||||
for (i in 0 until sampleLength) {
|
||||
val x = abs(samples[i * numChannels + index].toInt()) / VALUE_10
|
||||
val y = (if (x > 0) ((x - 1) / x.toFloat()) else x.toFloat())
|
||||
tempList.add(y)
|
||||
}
|
||||
|
||||
codec.releaseOutputBuffer(index, false)
|
||||
|
||||
// Cancels the process if it ends, exceeds the time limit, or the activity falls out of view
|
||||
val currTime = SystemClock.elapsedRealtime() - startTime
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 || currTime > TIME_LIMIT) {
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0 ||
|
||||
currTime > TIME_LIMIT ||
|
||||
currentLifeCycleFlag == LifeCycleFlag.PAUSED
|
||||
) {
|
||||
Log.d(
|
||||
TAG,
|
||||
"Processing ended with time: $currTime \n" +
|
||||
"Is finished: ${info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM > 0} \n" +
|
||||
"Lifecycle state: $currentLifeCycleFlag"
|
||||
)
|
||||
codec.stop()
|
||||
codec.release()
|
||||
extractor!!.release()
|
||||
extractor = null
|
||||
if (currTime < TIME_LIMIT) {
|
||||
result = tempList
|
||||
result = if (currTime < TIME_LIMIT) {
|
||||
tempList
|
||||
} else {
|
||||
Log.d(TAG, "time limit exceeded")
|
||||
Log.e(TAG, "Error in MediaCodec Callback:\n\tonOutputBufferAvailable: Time limit exceeded")
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -131,20 +195,30 @@ object AudioUtils {
|
||||
codec.release()
|
||||
extractor!!.release()
|
||||
extractor = null
|
||||
result = tempList
|
||||
result = null
|
||||
}
|
||||
|
||||
override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
|
||||
// unused atm
|
||||
}
|
||||
})
|
||||
|
||||
// More Boiler plate to start the codec
|
||||
mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
|
||||
mediaCodec.configure(mediaFormat, null, null, 0)
|
||||
mediaCodec.start()
|
||||
while (result.size <= 0) {
|
||||
|
||||
// This runs until the codec finishes or the time limit is exceeded, or an error occurs
|
||||
// If the time limit is exceed or an error occurs, the result should be null
|
||||
while (result != null && result!!.size <= 0) {
|
||||
continue
|
||||
}
|
||||
it.resume(shrinkFloatArray(result.toFloatArray(), DEFAULT_SIZE))
|
||||
|
||||
if (result != null && result!!.size > DEFAULT_SIZE) {
|
||||
it.resume(shrinkFloatArray(result!!.toFloatArray(), DEFAULT_SIZE))
|
||||
} else {
|
||||
it.resume(FloatArray(DEFAULT_SIZE))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user