Add helper class to broadcast the local participant state

The LocalStateBroadcaster observes changes in the
LocalCallParticipantModel and notifies other participants in the call as
needed. Although it is created right before joining the call there is a
slim chance of the state changing before the local participant is
actually in the call, but even in that case other participants would not
be notified about the state due to the MessageSender depending on the
list of call participants / peer connections passed to it, which should
not be initialized before the local participant is actually in the call.

There is, however, a race condition that could cause participants to not
be added to the participant list if they join at the same time as the
local participant and a signaling message listing them but not the local
participant as in the call is received once the CallParticipantList was
created, but that is unrelated to the broadcaster and will be fixed
in another commit.

Currently only changes in the audio, speaking and video state are
notified, although in the future it should also notify about the nick,
the raised hand or any other state (but not one-time events, like
reactions). The notifications right now are sent only through data
channels, but at a later point they will be sent also through signaling
messages as needed.

Similarly, although right now it only notifies of changes in the state
it will also take care of notifying other participants about the current
state when they join the call (or the local participant joins).

Signed-off-by: Daniel Calviño Sánchez <danxuliu@gmail.com>
This commit is contained in:
Daniel Calviño Sánchez 2024-11-30 06:25:26 +01:00 committed by Marcel Hibbe
parent cb52fb349f
commit fe32bc1628
No known key found for this signature in database
GPG Key ID: C793F8B59F43CE7B
3 changed files with 371 additions and 35 deletions

View File

@ -63,6 +63,7 @@ import com.nextcloud.talk.application.NextcloudTalkApplication.Companion.sharedA
import com.nextcloud.talk.call.CallParticipant
import com.nextcloud.talk.call.CallParticipantList
import com.nextcloud.talk.call.CallParticipantModel
import com.nextcloud.talk.call.LocalStateBroadcaster
import com.nextcloud.talk.call.MessageSender
import com.nextcloud.talk.call.MessageSenderMcu
import com.nextcloud.talk.call.MessageSenderNoMcu
@ -248,6 +249,7 @@ class CallActivity : CallBaseActivity() {
private var signalingMessageSender: SignalingMessageSender? = null
private var messageSender: MessageSender? = null
private val localCallParticipantModel: MutableLocalCallParticipantModel = MutableLocalCallParticipantModel()
private var localStateBroadcaster: LocalStateBroadcaster? = null
private val offerAnswerNickProviders: MutableMap<String?, OfferAnswerNickProvider?> = HashMap()
private val callParticipantMessageListeners: MutableMap<String?, CallParticipantMessageListener> = HashMap()
private val selfPeerConnectionObserver: PeerConnectionObserver = CallActivitySelfPeerConnectionObserver()
@ -1142,7 +1144,6 @@ class CallActivity : CallBaseActivity() {
@SuppressLint("MissingPermission")
private fun startMicInputDetection() {
if (permissionUtil!!.isMicrophonePermissionGranted() && micInputAudioRecordThread == null) {
var isSpeakingLongTerm = false
micInputAudioRecorder = AudioRecord(
MediaRecorder.AudioSource.MIC,
SAMPLE_RATE,
@ -1159,15 +1160,8 @@ class CallActivity : CallBaseActivity() {
micInputAudioRecorder.read(byteArr, 0, byteArr.size)
val isCurrentlySpeaking = abs(byteArr[0].toDouble()) > MICROPHONE_VALUE_THRESHOLD
if (microphoneOn && isCurrentlySpeaking && !isSpeakingLongTerm) {
isSpeakingLongTerm = true
localCallParticipantModel.isSpeaking = true
sendIsSpeakingMessage(true)
} else if (!isCurrentlySpeaking && isSpeakingLongTerm) {
isSpeakingLongTerm = false
localCallParticipantModel.isSpeaking = false
sendIsSpeakingMessage(false)
}
localCallParticipantModel.isSpeaking = isCurrentlySpeaking
Thread.sleep(MICROPHONE_VALUE_SLEEP)
}
}
@ -1176,16 +1170,6 @@ class CallActivity : CallBaseActivity() {
}
}
@Suppress("Detekt.NestedBlockDepth")
private fun sendIsSpeakingMessage(isSpeaking: Boolean) {
val isSpeakingMessage: String =
if (isSpeaking) SIGNALING_MESSAGE_SPEAKING_STARTED else SIGNALING_MESSAGE_SPEAKING_STOPPED
if (isConnectionEstablished && othersInCall) {
messageSender!!.sendToAll(DataChannelMessage(isSpeakingMessage))
}
}
private fun createCameraCapturer(enumerator: CameraEnumerator?): VideoCapturer? {
val deviceNames = enumerator!!.deviceNames
@ -1329,12 +1313,9 @@ class CallActivity : CallBaseActivity() {
}
private fun toggleMedia(enable: Boolean, video: Boolean) {
var message: String
if (video) {
message = SIGNALING_MESSAGE_VIDEO_OFF
if (enable) {
binding!!.cameraButton.alpha = OPACITY_ENABLED
message = SIGNALING_MESSAGE_VIDEO_ON
startVideoCapture()
} else {
binding!!.cameraButton.alpha = OPACITY_DISABLED
@ -1356,9 +1337,7 @@ class CallActivity : CallBaseActivity() {
binding!!.selfVideoRenderer.visibility = View.INVISIBLE
}
} else {
message = SIGNALING_MESSAGE_AUDIO_OFF
if (enable) {
message = SIGNALING_MESSAGE_AUDIO_ON
binding!!.microphoneButton.alpha = OPACITY_ENABLED
} else {
binding!!.microphoneButton.alpha = OPACITY_DISABLED
@ -1368,9 +1347,6 @@ class CallActivity : CallBaseActivity() {
localCallParticipantModel.isAudioEnabled = enable
}
}
if (isConnectionEstablished) {
messageSender!!.sendToAll(DataChannelMessage(message))
}
}
fun clickRaiseOrLowerHandButton() {
@ -1752,6 +1728,8 @@ class CallActivity : CallBaseActivity() {
callParticipantList = CallParticipantList(signalingMessageReceiver)
callParticipantList!!.addObserver(callParticipantListObserver)
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, messageSender)
val apiVersion = ApiUtils.getCallApiVersion(conversationUser, intArrayOf(ApiUtils.API_V4, 1))
ncApi!!.joinCall(
credentials,
@ -2104,6 +2082,9 @@ class CallActivity : CallBaseActivity() {
private fun hangupNetworkCalls(shutDownView: Boolean, endCallForAll: Boolean) {
Log.d(TAG, "hangupNetworkCalls. shutDownView=$shutDownView")
val apiVersion = ApiUtils.getCallApiVersion(conversationUser, intArrayOf(ApiUtils.API_V4, 1))
if (localStateBroadcaster != null) {
localStateBroadcaster!!.destroy()
}
if (callParticipantList != null) {
callParticipantList!!.removeObserver(callParticipantListObserver)
callParticipantList!!.destroy()
@ -3290,12 +3271,5 @@ class CallActivity : CallBaseActivity() {
private const val Y_POS_NO_CALL_INFO: Float = 20f
private const val SESSION_ID_PREFFIX_END: Int = 4
private const val SIGNALING_MESSAGE_SPEAKING_STARTED = "speaking"
private const val SIGNALING_MESSAGE_SPEAKING_STOPPED = "stoppedSpeaking"
private const val SIGNALING_MESSAGE_VIDEO_ON = "videoOn"
private const val SIGNALING_MESSAGE_VIDEO_OFF = "videoOff"
private const val SIGNALING_MESSAGE_AUDIO_ON = "audioOn"
private const val SIGNALING_MESSAGE_AUDIO_OFF = "audioOff"
}
}

View File

@ -0,0 +1,102 @@
/*
* Nextcloud Talk - Android Client
*
* SPDX-FileCopyrightText: 2024 Daniel Calviño Sánchez <danxuliu@gmail.com>
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package com.nextcloud.talk.call;
import com.nextcloud.talk.models.json.signaling.DataChannelMessage;
import java.util.Objects;
/**
* Helper class to send the local participant state to the other participants in the call.
* <p>
* Once created, and until destroyed, the LocalStateBroadcaster will send the changes in the local participant state to
* all the participants in the call. Note that the LocalStateBroadcaster does not check whether the local participant
* is actually in the call or not; it is expected that the LocalStateBroadcaster will be created and destroyed when the
* local participant joins and leaves the call.
*/
public class LocalStateBroadcaster {
private final LocalCallParticipantModel localCallParticipantModel;
private final LocalCallParticipantModelObserver localCallParticipantModelObserver;
private final MessageSender messageSender;
private class LocalCallParticipantModelObserver implements LocalCallParticipantModel.Observer {
private Boolean audioEnabled;
private Boolean speaking;
private Boolean videoEnabled;
public LocalCallParticipantModelObserver(LocalCallParticipantModel localCallParticipantModel) {
audioEnabled = localCallParticipantModel.isAudioEnabled();
speaking = localCallParticipantModel.isSpeaking();
videoEnabled = localCallParticipantModel.isVideoEnabled();
}
@Override
public void onChange() {
if (!Objects.equals(audioEnabled, localCallParticipantModel.isAudioEnabled())) {
audioEnabled = localCallParticipantModel.isAudioEnabled();
messageSender.sendToAll(getDataChannelMessageForAudioState());
}
if (!Objects.equals(speaking, localCallParticipantModel.isSpeaking())) {
speaking = localCallParticipantModel.isSpeaking();
messageSender.sendToAll(getDataChannelMessageForSpeakingState());
}
if (!Objects.equals(videoEnabled, localCallParticipantModel.isVideoEnabled())) {
videoEnabled = localCallParticipantModel.isVideoEnabled();
messageSender.sendToAll(getDataChannelMessageForVideoState());
}
}
}
public LocalStateBroadcaster(LocalCallParticipantModel localCallParticipantModel,
MessageSender messageSender) {
this.localCallParticipantModel = localCallParticipantModel;
this.localCallParticipantModelObserver = new LocalCallParticipantModelObserver(localCallParticipantModel);
this.messageSender = messageSender;
this.localCallParticipantModel.addObserver(localCallParticipantModelObserver);
}
public void destroy() {
this.localCallParticipantModel.removeObserver(localCallParticipantModelObserver);
}
private DataChannelMessage getDataChannelMessageForAudioState() {
String type = "audioOff";
if (localCallParticipantModel.isAudioEnabled() != null && localCallParticipantModel.isAudioEnabled()) {
type = "audioOn";
}
return new DataChannelMessage(type);
}
private DataChannelMessage getDataChannelMessageForSpeakingState() {
String type = "stoppedSpeaking";
if (localCallParticipantModel.isSpeaking() != null && localCallParticipantModel.isSpeaking()) {
type = "speaking";
}
return new DataChannelMessage(type);
}
private DataChannelMessage getDataChannelMessageForVideoState() {
String type = "videoOff";
if (localCallParticipantModel.isVideoEnabled() != null && localCallParticipantModel.isVideoEnabled()) {
type = "videoOn";
}
return new DataChannelMessage(type);
}
}

View File

@ -0,0 +1,260 @@
/*
* Nextcloud Talk - Android Client
*
* SPDX-FileCopyrightText: 2024 Daniel Calviño Sánchez <danxuliu@gmail.com>
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package com.nextcloud.talk.call
import com.nextcloud.talk.models.json.signaling.DataChannelMessage
import org.junit.Before
import org.junit.Test
import org.mockito.Mockito
@Suppress("TooManyFunctions")
class LocalStateBroadcasterTest {
private var localCallParticipantModel: MutableLocalCallParticipantModel? = null
private var mockedMessageSender: MessageSender? = null
private var localStateBroadcaster: LocalStateBroadcaster? = null
@Before
fun setUp() {
localCallParticipantModel = MutableLocalCallParticipantModel()
mockedMessageSender = Mockito.mock(MessageSender::class.java)
}
@Test
fun testEnableAudio() {
localCallParticipantModel!!.isAudioEnabled = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isAudioEnabled = true
val expectedAudioOn = DataChannelMessage("audioOn")
Mockito.verify(mockedMessageSender!!).sendToAll(expectedAudioOn)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testEnableAudioTwice() {
localCallParticipantModel!!.isAudioEnabled = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isAudioEnabled = true
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableAudio() {
localCallParticipantModel!!.isAudioEnabled = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isAudioEnabled = false
val expectedAudioOff = DataChannelMessage("audioOff")
Mockito.verify(mockedMessageSender!!).sendToAll(expectedAudioOff)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableAudioTwice() {
localCallParticipantModel!!.isAudioEnabled = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isAudioEnabled = false
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testEnableSpeaking() {
localCallParticipantModel!!.isAudioEnabled = true
localCallParticipantModel!!.isSpeaking = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = true
val expectedSpeaking = DataChannelMessage("speaking")
Mockito.verify(mockedMessageSender!!).sendToAll(expectedSpeaking)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testEnableSpeakingTwice() {
localCallParticipantModel!!.isAudioEnabled = true
localCallParticipantModel!!.isSpeaking = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = true
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testEnableSpeakingWithAudioDisabled() {
localCallParticipantModel!!.isAudioEnabled = false
localCallParticipantModel!!.isSpeaking = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = true
Mockito.verifyNoInteractions(mockedMessageSender)
}
@Test
fun testEnableAudioWhileSpeaking() {
localCallParticipantModel!!.isAudioEnabled = false
localCallParticipantModel!!.isSpeaking = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = true
localCallParticipantModel!!.isAudioEnabled = true
val expectedAudioOn = DataChannelMessage("audioOn")
val expectedSpeaking = DataChannelMessage("speaking")
val inOrder = Mockito.inOrder(mockedMessageSender)
inOrder.verify(mockedMessageSender!!).sendToAll(expectedAudioOn)
inOrder.verify(mockedMessageSender!!).sendToAll(expectedSpeaking)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableSpeaking() {
localCallParticipantModel!!.isAudioEnabled = true
localCallParticipantModel!!.isSpeaking = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = false
val expectedStoppedSpeaking = DataChannelMessage("stoppedSpeaking")
Mockito.verify(mockedMessageSender!!).sendToAll(expectedStoppedSpeaking)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableSpeakingTwice() {
localCallParticipantModel!!.isAudioEnabled = true
localCallParticipantModel!!.isSpeaking = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = false
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableAudioWhileSpeaking() {
localCallParticipantModel!!.isAudioEnabled = true
localCallParticipantModel!!.isSpeaking = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isAudioEnabled = false
val expectedStoppedSpeaking = DataChannelMessage("stoppedSpeaking")
val expectedAudioOff = DataChannelMessage("audioOff")
val inOrder = Mockito.inOrder(mockedMessageSender)
inOrder.verify(mockedMessageSender!!).sendToAll(expectedStoppedSpeaking)
inOrder.verify(mockedMessageSender!!).sendToAll(expectedAudioOff)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableSpeakingWithAudioDisabled() {
localCallParticipantModel!!.isAudioEnabled = false
localCallParticipantModel!!.isSpeaking = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isSpeaking = false
Mockito.verifyNoInteractions(mockedMessageSender)
}
@Test
fun testEnableVideo() {
localCallParticipantModel!!.isVideoEnabled = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isVideoEnabled = true
val expectedVideoOn = DataChannelMessage("videoOn")
Mockito.verify(mockedMessageSender!!).sendToAll(expectedVideoOn)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testEnableVideoTwice() {
localCallParticipantModel!!.isVideoEnabled = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isVideoEnabled = true
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableVideo() {
localCallParticipantModel!!.isVideoEnabled = true
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isVideoEnabled = false
val expectedVideoOff = DataChannelMessage("videoOff")
Mockito.verify(mockedMessageSender!!).sendToAll(expectedVideoOff)
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testDisableVideoTwice() {
localCallParticipantModel!!.isVideoEnabled = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localCallParticipantModel!!.isVideoEnabled = false
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
@Test
fun testChangeStateAfterDestroying() {
localCallParticipantModel!!.isAudioEnabled = false
localCallParticipantModel!!.isSpeaking = false
localCallParticipantModel!!.isVideoEnabled = false
localStateBroadcaster = LocalStateBroadcaster(localCallParticipantModel, mockedMessageSender)
localStateBroadcaster!!.destroy()
localCallParticipantModel!!.isAudioEnabled = true
localCallParticipantModel!!.isSpeaking = true
localCallParticipantModel!!.isVideoEnabled = true
Mockito.verifyNoMoreInteractions(mockedMessageSender)
}
}