From 175944e9323a3c2ac0dd4a70700539daf17fdfd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 26 Nov 2022 04:18:09 +0100 Subject: [PATCH] Move handling of call participants to its own class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CallParticipant provides a read-only CallParticipantModel and internally handles the data channel and peer connection events that modify the model. Nevertheless, the CallParticipant requires certain properties to be externally set, like the userId or the peer connections. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 133 +++--------- .../nextcloud/talk/call/CallParticipant.java | 198 ++++++++++++++++++ 2 files changed, 224 insertions(+), 107 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipant.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 321b8257b..ab5a79384 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -60,8 +60,8 @@ import com.nextcloud.talk.adapters.ParticipantDisplayItem; import com.nextcloud.talk.adapters.ParticipantsAdapter; import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; +import com.nextcloud.talk.call.CallParticipant; import com.nextcloud.talk.call.CallParticipantModel; -import com.nextcloud.talk.call.MutableCallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; import com.nextcloud.talk.events.ConfigurationChangeEvent; @@ -264,11 +264,9 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); - private Map dataChannelMessageListeners = new HashMap<>(); - private Map peerConnectionObservers = new HashMap<>(); - private Map callParticipantModels = new HashMap<>(); + private Map callParticipants = new HashMap<>(); private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @@ -385,7 +383,7 @@ public class CallActivity extends CallBaseActivity { requestBluetoothPermission(); } basicInitialization(); - callParticipantModels = new HashMap<>(); + callParticipants = new HashMap<>(); participantDisplayItems = new HashMap<>(); initViews(); if (!isConnectionEstablished()) { @@ -1860,7 +1858,7 @@ public class CallActivity extends CallBaseActivity { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { - callParticipantModels.get(sessionId).setUserId(userId); + callParticipants.get(sessionId).setUserId(userId); } String nick; @@ -1869,7 +1867,7 @@ public class CallActivity extends CallBaseActivity { } else { nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; } - callParticipantModels.get(sessionId).setNick(nick); + callParticipants.get(sessionId).setNick(nick); } if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { @@ -1975,12 +1973,6 @@ public class CallActivity extends CallBaseActivity { new CallActivityCallParticipantMessageListener(sessionId); callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - - // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. - PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = - new CallActivityDataChannelMessageListener(sessionId); - dataChannelMessageListeners.put(sessionId, dataChannelMessageListener); - peerConnectionWrapper.addListener(dataChannelMessageListener); } if (!publisher && !hasExternalSignalingServer && offerAnswerNickProviders.get(sessionId) == null) { @@ -1996,13 +1988,19 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapper.addObserver(peerConnectionObserver); if (!publisher) { - MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); - if (mutableCallParticipantModel == null) { - mutableCallParticipantModel = new MutableCallParticipantModel(sessionId); - callParticipantModels.put(sessionId, mutableCallParticipantModel); + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant == null) { + callParticipant = new CallParticipant(sessionId); + callParticipants.put(sessionId, callParticipant); } - final CallParticipantModel callParticipantModel = mutableCallParticipantModel; + if ("screen".equals(type)) { + callParticipant.setScreenPeerConnectionWrapper(peerConnectionWrapper); + } else { + callParticipant.setPeerConnectionWrapper(peerConnectionWrapper); + } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); runOnUiThread(() -> { setupVideoStreamForLayout(callParticipantModel, type); @@ -2033,10 +2031,6 @@ public class CallActivity extends CallBaseActivity { if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - if (!justScreen && VIDEO_STREAM_TYPE_VIDEO.equals(peerConnectionWrapper.getVideoStreamType())) { - PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = dataChannelMessageListeners.remove(sessionId); - peerConnectionWrapper.removeListener(dataChannelMessageListener); - } String videoStreamType = peerConnectionWrapper.getVideoStreamType(); if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); @@ -2044,16 +2038,12 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); - MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); - if (mutableCallParticipantModel != null) { + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant != null) { if ("screen".equals(videoStreamType)) { - mutableCallParticipantModel.setScreenMediaStream(null); - mutableCallParticipantModel.setScreenIceConnectionState(null); + callParticipant.setScreenPeerConnectionWrapper(null); } else { - mutableCallParticipantModel.setMediaStream(null); - mutableCallParticipantModel.setIceConnectionState(null); - mutableCallParticipantModel.setAudioAvailable(null); - mutableCallParticipantModel.setVideoAvailable(null); + callParticipant.setPeerConnectionWrapper(null); } } @@ -2073,7 +2063,10 @@ public class CallActivity extends CallBaseActivity { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } - callParticipantModels.remove(sessionId); + CallParticipant callParticipant = callParticipants.remove(sessionId); + if (callParticipant != null) { + callParticipant.destroy(); + } } } @@ -2529,8 +2522,8 @@ public class CallActivity extends CallBaseActivity { private void onOfferOrAnswer(String nick) { this.nick = nick; - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setNick(nick); + if (callParticipants.get(sessionId) != null) { + callParticipants.get(sessionId).setNick(nick); } } @@ -2561,50 +2554,6 @@ public class CallActivity extends CallBaseActivity { } } - private class CallActivityDataChannelMessageListener implements PeerConnectionWrapper.DataChannelMessageListener { - - private final String sessionId; - - private CallActivityDataChannelMessageListener(String sessionId) { - this.sessionId = sessionId; - } - - @Override - public void onAudioOn() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setAudioAvailable(true); - } - } - - @Override - public void onAudioOff() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setAudioAvailable(false); - } - } - - @Override - public void onVideoOn() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setVideoAvailable(true); - } - } - - @Override - public void onVideoOff() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setVideoAvailable(false); - } - } - - @Override - public void onNickChanged(String nick) { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setNick(nick); - } - } - } - private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { private final String sessionId; @@ -2617,44 +2566,14 @@ public class CallActivity extends CallBaseActivity { @Override public void onStreamAdded(MediaStream mediaStream) { - handleStream(mediaStream); } @Override public void onStreamRemoved(MediaStream mediaStream) { - handleStream(null); - } - - private void handleStream(MediaStream mediaStream) { - if (callParticipantModels.get(sessionId) == null) { - return; - } - - if ("screen".equals(videoStreamType)) { - callParticipantModels.get(sessionId).setScreenMediaStream(mediaStream); - - return; - } - - boolean hasAtLeastOneVideoStream = false; - if (mediaStream != null) { - hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; - } - - callParticipantModels.get(sessionId).setMediaStream(mediaStream); - callParticipantModels.get(sessionId).setVideoAvailable(hasAtLeastOneVideoStream); } @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { - if (callParticipantModels.get(sessionId) != null) { - if ("screen".equals(videoStreamType)) { - callParticipantModels.get(sessionId).setScreenIceConnectionState(iceConnectionState); - } else { - callParticipantModels.get(sessionId).setIceConnectionState(iceConnectionState); - } - } - runOnUiThread(() -> { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { updateSelfVideoViewIceConnectionState(iceConnectionState); diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java new file mode 100644 index 000000000..3b153f8cb --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java @@ -0,0 +1,198 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.webrtc.PeerConnectionWrapper; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +/** + * Model for (remote) call participants. + * + * This class keeps track of the state changes in a call participant and updates its data model as needed. View classes + * are expected to directly use the read-only data model. + */ +public class CallParticipant { + + private final PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = + new PeerConnectionWrapper.PeerConnectionObserver() { + @Override + public void onStreamAdded(MediaStream mediaStream) { + handleStreamChange(mediaStream); + } + + @Override + public void onStreamRemoved(MediaStream mediaStream) { + handleStreamChange(mediaStream); + } + + @Override + public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + handleIceConnectionStateChange(iceConnectionState); + } + }; + + private final PeerConnectionWrapper.PeerConnectionObserver screenPeerConnectionObserver = + new PeerConnectionWrapper.PeerConnectionObserver() { + @Override + public void onStreamAdded(MediaStream mediaStream) { + callParticipantModel.setScreenMediaStream(mediaStream); + } + + @Override + public void onStreamRemoved(MediaStream mediaStream) { + callParticipantModel.setScreenMediaStream(null); + } + + @Override + public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + callParticipantModel.setScreenIceConnectionState(iceConnectionState); + } + }; + + // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. + private final PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = + new PeerConnectionWrapper.DataChannelMessageListener() { + @Override + public void onAudioOn() { + callParticipantModel.setAudioAvailable(Boolean.TRUE); + } + + @Override + public void onAudioOff() { + callParticipantModel.setAudioAvailable(Boolean.FALSE); + } + + @Override + public void onVideoOn() { + callParticipantModel.setVideoAvailable(Boolean.TRUE); + } + + @Override + public void onVideoOff() { + callParticipantModel.setVideoAvailable(Boolean.FALSE); + } + + @Override + public void onNickChanged(String nick) { + callParticipantModel.setNick(nick); + } + }; + + private final MutableCallParticipantModel callParticipantModel; + + private PeerConnectionWrapper peerConnectionWrapper; + private PeerConnectionWrapper screenPeerConnectionWrapper; + + public CallParticipant(String sessionId) { + callParticipantModel = new MutableCallParticipantModel(sessionId); + } + + public void destroy() { + if (peerConnectionWrapper != null) { + peerConnectionWrapper.removeObserver(peerConnectionObserver); + peerConnectionWrapper.removeListener(dataChannelMessageListener); + } + if (screenPeerConnectionWrapper != null) { + screenPeerConnectionWrapper.removeObserver(screenPeerConnectionObserver); + } + } + + public CallParticipantModel getCallParticipantModel() { + return callParticipantModel; + } + + public void setUserId(String userId) { + callParticipantModel.setUserId(userId); + } + + public void setNick(String nick) { + callParticipantModel.setNick(nick); + } + + public void setPeerConnectionWrapper(PeerConnectionWrapper peerConnectionWrapper) { + if (this.peerConnectionWrapper != null) { + this.peerConnectionWrapper.removeObserver(peerConnectionObserver); + this.peerConnectionWrapper.removeListener(dataChannelMessageListener); + } + + this.peerConnectionWrapper = peerConnectionWrapper; + + if (this.peerConnectionWrapper == null) { + callParticipantModel.setIceConnectionState(null); + callParticipantModel.setMediaStream(null); + callParticipantModel.setAudioAvailable(null); + callParticipantModel.setVideoAvailable(null); + + return; + } + + handleIceConnectionStateChange(this.peerConnectionWrapper.getPeerConnection().iceConnectionState()); + handleStreamChange(this.peerConnectionWrapper.getStream()); + + this.peerConnectionWrapper.addObserver(peerConnectionObserver); + this.peerConnectionWrapper.addListener(dataChannelMessageListener); + } + + private void handleIceConnectionStateChange(PeerConnection.IceConnectionState iceConnectionState) { + callParticipantModel.setIceConnectionState(iceConnectionState); + + if (iceConnectionState == PeerConnection.IceConnectionState.NEW || + iceConnectionState == PeerConnection.IceConnectionState.CHECKING) { + callParticipantModel.setAudioAvailable(null); + callParticipantModel.setVideoAvailable(null); + } + } + + private void handleStreamChange(MediaStream mediaStream) { + if (mediaStream == null) { + callParticipantModel.setMediaStream(null); + callParticipantModel.setVideoAvailable(Boolean.FALSE); + + return; + } + + boolean hasAtLeastOneVideoStream = mediaStream.videoTracks != null && !mediaStream.videoTracks.isEmpty(); + + callParticipantModel.setMediaStream(mediaStream); + callParticipantModel.setVideoAvailable(hasAtLeastOneVideoStream); + } + + public void setScreenPeerConnectionWrapper(PeerConnectionWrapper screenPeerConnectionWrapper) { + if (this.screenPeerConnectionWrapper != null) { + this.screenPeerConnectionWrapper.removeObserver(screenPeerConnectionObserver); + } + + this.screenPeerConnectionWrapper = screenPeerConnectionWrapper; + + if (this.screenPeerConnectionWrapper == null) { + callParticipantModel.setScreenIceConnectionState(null); + callParticipantModel.setScreenMediaStream(null); + + return; + } + + callParticipantModel.setScreenIceConnectionState(this.screenPeerConnectionWrapper.getPeerConnection().iceConnectionState()); + callParticipantModel.setScreenMediaStream(this.screenPeerConnectionWrapper.getStream()); + + this.screenPeerConnectionWrapper.addObserver(screenPeerConnectionObserver); + } +}