From 7b1bfb2c178fa60135112a1f963fd37c11deac7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:30:46 +0200 Subject: [PATCH 01/14] Simplify getting the message type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The message type is set for all signaling messages. On the other hand, the payload type is only set for offers and answers (and, if the message was sent by the Android app, also for candidates). However, in all those cases the payload type just duplicates the message type, so the message type can be assigned directly rather than falling back to it if there is no payload type. Signed-off-by: Daniel Calviño Sánchez --- .../java/com/nextcloud/talk/activities/CallActivity.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 58cb13ea0..78c21dec4 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1664,12 +1664,7 @@ public class CallActivity extends CallBaseActivity { private void processMessage(NCSignalingMessage ncSignalingMessage) { if ("video".equals(ncSignalingMessage.getRoomType()) || "screen".equals(ncSignalingMessage.getRoomType())) { - String type = null; - if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != null) { - type = ncSignalingMessage.getPayload().getType(); - } else if (ncSignalingMessage.getType() != null) { - type = ncSignalingMessage.getType(); - } + String type = ncSignalingMessage.getType(); PeerConnectionWrapper peerConnectionWrapper = null; From 075ba20cf926577d849daf22f79b640c7eeaed67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:34:43 +0200 Subject: [PATCH 02/14] Split handling of "unshareScreen" and WebRTC messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../nextcloud/talk/activities/CallActivity.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 78c21dec4..25e50d6db 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1666,6 +1666,12 @@ public class CallActivity extends CallBaseActivity { if ("video".equals(ncSignalingMessage.getRoomType()) || "screen".equals(ncSignalingMessage.getRoomType())) { String type = ncSignalingMessage.getType(); + if ("unshareScreen".equals(type)) { + endPeerConnection(ncSignalingMessage.getFrom(), true); + + return; + } + PeerConnectionWrapper peerConnectionWrapper = null; if ("offer".equals(type)) { @@ -1678,16 +1684,12 @@ public class CallActivity extends CallBaseActivity { ncSignalingMessage.getRoomType()); } - if ("unshareScreen".equals(type) || - (("offer".equals(type) || + if (("offer".equals(type) || "answer".equals(type) || "candidate".equals(type) || "endOfCandidates".equals(type)) && - peerConnectionWrapper != null)) { + peerConnectionWrapper != null) { switch (type) { - case "unshareScreen": - endPeerConnection(ncSignalingMessage.getFrom(), true); - break; case "offer": case "answer": peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); From f3e04b8e18914bc680c292306c859129c4f7ed29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:40:00 +0200 Subject: [PATCH 03/14] Simplify condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "peerConnectionWrapper" needs to be defined to enter the if and execute the switch, so just return before the switch if "peerConnectionWrapper" is null. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 25e50d6db..5f04f4920 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1684,44 +1684,42 @@ public class CallActivity extends CallBaseActivity { ncSignalingMessage.getRoomType()); } - if (("offer".equals(type) || - "answer".equals(type) || - "candidate".equals(type) || - "endOfCandidates".equals(type)) && - peerConnectionWrapper != null) { - switch (type) { - case "offer": - case "answer": - peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - SessionDescription sessionDescriptionWithPreferredCodec; + if (peerConnectionWrapper == null) { + return; + } - String sessionDescriptionStringWithPreferredCodec = MagicWebRTCUtils.preferCodec - (ncSignalingMessage.getPayload().getSdp(), - "H264", false); + switch (type) { + case "offer": + case "answer": + peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); + SessionDescription sessionDescriptionWithPreferredCodec; - sessionDescriptionWithPreferredCodec = new SessionDescription( - SessionDescription.Type.fromCanonicalForm(type), - sessionDescriptionStringWithPreferredCodec); + String sessionDescriptionStringWithPreferredCodec = MagicWebRTCUtils.preferCodec + (ncSignalingMessage.getPayload().getSdp(), + "H264", false); - if (peerConnectionWrapper.getPeerConnection() != null) { - peerConnectionWrapper.getPeerConnection().setRemoteDescription( - peerConnectionWrapper.getMagicSdpObserver(), - sessionDescriptionWithPreferredCodec); - } - break; - case "candidate": - NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); - IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), - ncIceCandidate.getSdpMLineIndex(), - ncIceCandidate.getCandidate()); - peerConnectionWrapper.addCandidate(iceCandidate); - break; - case "endOfCandidates": - peerConnectionWrapper.drainIceCandidates(); - break; - default: - break; - } + sessionDescriptionWithPreferredCodec = new SessionDescription( + SessionDescription.Type.fromCanonicalForm(type), + sessionDescriptionStringWithPreferredCodec); + + if (peerConnectionWrapper.getPeerConnection() != null) { + peerConnectionWrapper.getPeerConnection().setRemoteDescription( + peerConnectionWrapper.getMagicSdpObserver(), + sessionDescriptionWithPreferredCodec); + } + break; + case "candidate": + NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); + IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), + ncIceCandidate.getSdpMLineIndex(), + ncIceCandidate.getCandidate()); + peerConnectionWrapper.addCandidate(iceCandidate); + break; + case "endOfCandidates": + peerConnectionWrapper.drainIceCandidates(); + break; + default: + break; } } else { Log.e(TAG, "unexpected RoomType while processing NCSignalingMessage"); From 4dffd29ceba62166390226165223137af3565d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:45:38 +0200 Subject: [PATCH 04/14] Move handling of WebRTC messages to PeerConnectionWrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 33 +++++--------- .../talk/webrtc/PeerConnectionWrapper.java | 43 +++++++++++++++++++ 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 5f04f4920..33528f640 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -116,14 +116,12 @@ import org.webrtc.CameraVideoCapturer; import org.webrtc.DefaultVideoDecoderFactory; import org.webrtc.DefaultVideoEncoderFactory; import org.webrtc.EglBase; -import org.webrtc.IceCandidate; import org.webrtc.Logging; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.RendererCommon; -import org.webrtc.SessionDescription; import org.webrtc.SurfaceTextureHelper; import org.webrtc.VideoCapturer; import org.webrtc.VideoSource; @@ -1688,35 +1686,24 @@ public class CallActivity extends CallBaseActivity { return; } + String sdp = ncSignalingMessage.getPayload().getSdp(); + String nick = ncSignalingMessage.getPayload().getNick(); + switch (type) { case "offer": + peerConnectionWrapper.getWebRtcMessageListener().onOffer(sdp, nick); + break; case "answer": - peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - SessionDescription sessionDescriptionWithPreferredCodec; - - String sessionDescriptionStringWithPreferredCodec = MagicWebRTCUtils.preferCodec - (ncSignalingMessage.getPayload().getSdp(), - "H264", false); - - sessionDescriptionWithPreferredCodec = new SessionDescription( - SessionDescription.Type.fromCanonicalForm(type), - sessionDescriptionStringWithPreferredCodec); - - if (peerConnectionWrapper.getPeerConnection() != null) { - peerConnectionWrapper.getPeerConnection().setRemoteDescription( - peerConnectionWrapper.getMagicSdpObserver(), - sessionDescriptionWithPreferredCodec); - } + peerConnectionWrapper.getWebRtcMessageListener().onAnswer(sdp, nick); break; case "candidate": NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); - IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), - ncIceCandidate.getSdpMLineIndex(), - ncIceCandidate.getCandidate()); - peerConnectionWrapper.addCandidate(iceCandidate); + peerConnectionWrapper.getWebRtcMessageListener().onCandidate(ncIceCandidate.getSdpMid(), + ncIceCandidate.getSdpMLineIndex(), + ncIceCandidate.getCandidate()); break; case "endOfCandidates": - peerConnectionWrapper.drainIceCandidates(); + peerConnectionWrapper.getWebRtcMessageListener().onEndOfCandidates(); break; default: break; diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index f0597f5a7..ad0a9744a 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -74,6 +74,8 @@ public class PeerConnectionWrapper { private static final String TAG = PeerConnectionWrapper.class.getCanonicalName(); + private final WebRtcMessageListener webRtcMessageListener = new WebRtcMessageListener(); + private List iceCandidates = new ArrayList<>(); private PeerConnection peerConnection; private String sessionId; @@ -267,6 +269,47 @@ public class PeerConnectionWrapper { return false; } + public WebRtcMessageListener getWebRtcMessageListener() { + return webRtcMessageListener; + } + + public class WebRtcMessageListener { + + public void onOffer(String sdp, String nick) { + onOfferOrAnswer("offer", sdp, nick); + } + + public void onAnswer(String sdp, String nick) { + onOfferOrAnswer("answer", sdp, nick); + } + + private void onOfferOrAnswer(String type, String sdp, String nick) { + setNick(nick); + + SessionDescription sessionDescriptionWithPreferredCodec; + + boolean isAudio = false; + String sessionDescriptionStringWithPreferredCodec = MagicWebRTCUtils.preferCodec(sdp, "H264", isAudio); + + sessionDescriptionWithPreferredCodec = new SessionDescription( + SessionDescription.Type.fromCanonicalForm(type), + sessionDescriptionStringWithPreferredCodec); + + if (getPeerConnection() != null) { + getPeerConnection().setRemoteDescription(magicSdpObserver, sessionDescriptionWithPreferredCodec); + } + } + + public void onCandidate(String sdpMid, int sdpMLineIndex, String sdp) { + IceCandidate iceCandidate = new IceCandidate(sdpMid, sdpMLineIndex, sdp); + addCandidate(iceCandidate); + } + + public void onEndOfCandidates() { + drainIceCandidates(); + } + } + private class MagicDataChannelObserver implements DataChannel.Observer { @Override From 0e360020364da9ae3bb649adf57220ed158b9643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:46:22 +0200 Subject: [PATCH 05/14] Hide and delete no longer needed public methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the rest of the methods are no longer needed since the handling of WebRTC messages was moved to PeerConnectionWrapper "setSessionId()" was not needed even before that. Signed-off-by: Daniel Calviño Sánchez --- .../talk/webrtc/PeerConnectionWrapper.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index ad0a9744a..f4fbbeba9 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -169,7 +169,7 @@ public class PeerConnectionWrapper { } } - public void drainIceCandidates() { + private void drainIceCandidates() { if (peerConnection != null) { for (IceCandidate iceCandidate : iceCandidates) { @@ -180,11 +180,7 @@ public class PeerConnectionWrapper { } } - public MagicSdpObserver getMagicSdpObserver() { - return magicSdpObserver; - } - - public void addCandidate(IceCandidate iceCandidate) { + private void addCandidate(IceCandidate iceCandidate) { if (peerConnection != null && peerConnection.getRemoteDescription() != null) { peerConnection.addIceCandidate(iceCandidate); } else { @@ -224,10 +220,6 @@ public class PeerConnectionWrapper { return sessionId; } - public void setSessionId(String sessionId) { - this.sessionId = sessionId; - } - public String getNick() { if (!TextUtils.isEmpty(nick)) { return nick; @@ -236,7 +228,7 @@ public class PeerConnectionWrapper { } } - public void setNick(String nick) { + private void setNick(String nick) { this.nick = nick; } From 45787caf0a8a35d08cb36ab9517d683e0ad874f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:49:20 +0200 Subject: [PATCH 06/14] Add SignalingMessageReceiver class to listen to signaling messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now only WebRTC messages can be listened to, although it will be extended with other kinds later. This commit only introduces the base class, although it is not used yet anywhere; a concrete implementation will be added in a following commit. The test class is named "SignalingMessageReceiverWebRtcTest" rather than just "SignalingMessageReceiverTest" to have smaller, more manageable test classes for each listener kind rather than one large test class for all of them. Signed-off-by: Daniel Calviño Sánchez --- .../signaling/SignalingMessageReceiver.java | 222 +++++++++++ .../talk/signaling/WebRtcMessageNotifier.java | 120 ++++++ .../SignalingMessageReceiverWebRtcTest.java | 366 ++++++++++++++++++ 3 files changed, 708 insertions(+) create mode 100644 app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java create mode 100644 app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java create mode 100644 app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverWebRtcTest.java diff --git a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java new file mode 100644 index 000000000..f798ba9b8 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java @@ -0,0 +1,222 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.signaling.NCIceCandidate; +import com.nextcloud.talk.models.json.signaling.NCMessagePayload; +import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; + +/** + * Hub to register listeners for signaling messages of different kinds. + * + * Adding and removing listeners, as well as notifying them is internally synchronized. This should be kept in mind + * if listeners are added or removed when handling an event to prevent deadlocks (nevertheless, just adding or + * removing a listener in the same thread handling the event is fine, and in most cases it will be fine too if done + * in a different thread, as long as the notifier thread is not forced to wait until the listener is added or removed). + * + * SignalingMessageReceiver does not fetch the signaling messages itself; subclasses must fetch them and then call + * the appropriate protected methods to process the messages and notify the listeners. + */ +public abstract class SignalingMessageReceiver { + + /** + * Listener for WebRTC messages. + * + * The messages are bound to a specific peer connection, so each listener is expected to handle messages only for + * a single peer connection. + */ + public interface WebRtcMessageListener { + void onOffer(String sdp, String nick); + void onAnswer(String sdp, String nick); + void onCandidate(String sdpMid, int sdpMLineIndex, String sdp); + void onEndOfCandidates(); + } + + private final WebRtcMessageNotifier webRtcMessageNotifier = new WebRtcMessageNotifier(); + + /** + * Adds a listener for WebRTC messages from the given session ID and room type. + * + * A listener is expected to be added only once. If the same listener is added again it will no longer be notified + * for the messages from the previous session ID or room type. + * + * @param listener the WebRtcMessageListener + * @param sessionId the ID of the session that messages come from + * @param roomType the room type that messages come from + */ + public void addListener(WebRtcMessageListener listener, String sessionId, String roomType) { + webRtcMessageNotifier.addListener(listener, sessionId, roomType); + } + + public void removeListener(WebRtcMessageListener listener) { + webRtcMessageNotifier.removeListener(listener); + } + + protected void processSignalingMessage(NCSignalingMessage signalingMessage) { + // Note that in the internal signaling server message "data" is the String representation of a JSON + // object, although it is already decoded when used here. + + String type = signalingMessage.getType(); + + String sessionId = signalingMessage.getFrom(); + String roomType = signalingMessage.getRoomType(); + + if ("offer".equals(type)) { + // Message schema (external signaling server): + // { + // "type": "message", + // "message": { + // "sender": { + // ... + // }, + // "data": { + // "to": #STRING#, + // "from": #STRING#, + // "type": "offer", + // "roomType": #STRING#, // "video" or "screen" + // "payload": { + // "type": "offer", + // "sdp": #STRING#, + // }, + // "sid": #STRING#, // external signaling server >= 0.5.0 + // }, + // }, + // } + // + // Message schema (internal signaling server): + // { + // "type": "message", + // "data": { + // "to": #STRING#, + // "sid": #STRING#, + // "roomType": #STRING#, // "video" or "screen" + // "type": "offer", + // "payload": { + // "type": "offer", + // "sdp": #STRING#, + // "nick": #STRING#, // Optional + // }, + // "from": #STRING#, + // }, + // } + + NCMessagePayload payload = signalingMessage.getPayload(); + if (payload == null) { + // Broken message, this should not happen. + return; + } + + String sdp = payload.getSdp(); + String nick = payload.getNick(); + + webRtcMessageNotifier.notifyOffer(sessionId, roomType, sdp, nick); + + return; + } + + if ("answer".equals(type)) { + // Message schema: same as offers, but with type "answer". + + NCMessagePayload payload = signalingMessage.getPayload(); + if (payload == null) { + // Broken message, this should not happen. + return; + } + + String sdp = payload.getSdp(); + String nick = payload.getNick(); + + webRtcMessageNotifier.notifyAnswer(sessionId, roomType, sdp, nick); + + return; + } + + if ("candidate".equals(type)) { + // Message schema (external signaling server): + // { + // "type": "message", + // "message": { + // "sender": { + // ... + // }, + // "data": { + // "to": #STRING#, + // "from": #STRING#, + // "type": "candidate", + // "roomType": #STRING#, // "video" or "screen" + // "payload": { + // "candidate": { + // "candidate": #STRING#, + // "sdpMid": #STRING#, + // "sdpMLineIndex": #INTEGER#, + // }, + // }, + // "sid": #STRING#, // external signaling server >= 0.5.0 + // }, + // }, + // } + // + // Message schema (internal signaling server): + // { + // "type": "message", + // "data": { + // "to": #STRING#, + // "sid": #STRING#, + // "roomType": #STRING#, // "video" or "screen" + // "type": "candidate", + // "payload": { + // "candidate": { + // "candidate": #STRING#, + // "sdpMid": #STRING#, + // "sdpMLineIndex": #INTEGER#, + // }, + // }, + // "from": #STRING#, + // }, + // } + + NCMessagePayload payload = signalingMessage.getPayload(); + if (payload == null) { + // Broken message, this should not happen. + return; + } + + NCIceCandidate ncIceCandidate = payload.getIceCandidate(); + if (ncIceCandidate == null) { + // Broken message, this should not happen. + return; + } + + webRtcMessageNotifier.notifyCandidate(sessionId, + roomType, + ncIceCandidate.getSdpMid(), + ncIceCandidate.getSdpMLineIndex(), + ncIceCandidate.getCandidate()); + + return; + } + + if ("endOfCandidates".equals(type)) { + webRtcMessageNotifier.notifyEndOfCandidates(sessionId, roomType); + + return; + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java new file mode 100644 index 000000000..9dbe6cb38 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/signaling/WebRtcMessageNotifier.java @@ -0,0 +1,120 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Helper class to register and notify WebRtcMessageListeners. + * + * This class is only meant for internal use by SignalingMessageReceiver; listeners must register themselves against + * a SignalingMessageReceiver rather than against a WebRtcMessageNotifier. + */ +class WebRtcMessageNotifier { + + /** + * Helper class to associate a WebRtcMessageListener with a session ID and room type. + */ + private static class WebRtcMessageListenerFrom { + public final SignalingMessageReceiver.WebRtcMessageListener listener; + public final String sessionId; + public final String roomType; + + private WebRtcMessageListenerFrom(SignalingMessageReceiver.WebRtcMessageListener listener, + String sessionId, + String roomType) { + this.listener = listener; + this.sessionId = sessionId; + this.roomType = roomType; + } + } + + private final List webRtcMessageListenersFrom = new ArrayList<>(); + + public synchronized void addListener(SignalingMessageReceiver.WebRtcMessageListener listener, String sessionId, String roomType) { + if (listener == null) { + throw new IllegalArgumentException("WebRtcMessageListener can not be null"); + } + + if (sessionId == null) { + throw new IllegalArgumentException("sessionId can not be null"); + } + + if (roomType == null) { + throw new IllegalArgumentException("roomType can not be null"); + } + + removeListener(listener); + + webRtcMessageListenersFrom.add(new WebRtcMessageListenerFrom(listener, sessionId, roomType)); + } + + public synchronized void removeListener(SignalingMessageReceiver.WebRtcMessageListener listener) { + Iterator it = webRtcMessageListenersFrom.iterator(); + while (it.hasNext()) { + WebRtcMessageListenerFrom listenerFrom = it.next(); + + if (listenerFrom.listener == listener) { + it.remove(); + + return; + } + } + } + + private List getListenersFor(String sessionId, String roomType) { + List webRtcMessageListeners = + new ArrayList<>(webRtcMessageListenersFrom.size()); + + for (WebRtcMessageListenerFrom listenerFrom : webRtcMessageListenersFrom) { + if (listenerFrom.sessionId.equals(sessionId) && listenerFrom.roomType.equals(roomType)) { + webRtcMessageListeners.add(listenerFrom.listener); + } + } + + return webRtcMessageListeners; + } + + public synchronized void notifyOffer(String sessionId, String roomType, String sdp, String nick) { + for (SignalingMessageReceiver.WebRtcMessageListener listener : getListenersFor(sessionId, roomType)) { + listener.onOffer(sdp, nick); + } + } + + public synchronized void notifyAnswer(String sessionId, String roomType, String sdp, String nick) { + for (SignalingMessageReceiver.WebRtcMessageListener listener : getListenersFor(sessionId, roomType)) { + listener.onAnswer(sdp, nick); + } + } + + public synchronized void notifyCandidate(String sessionId, String roomType, String sdpMid, int sdpMLineIndex, String sdp) { + for (SignalingMessageReceiver.WebRtcMessageListener listener : getListenersFor(sessionId, roomType)) { + listener.onCandidate(sdpMid, sdpMLineIndex, sdp); + } + } + + public synchronized void notifyEndOfCandidates(String sessionId, String roomType) { + for (SignalingMessageReceiver.WebRtcMessageListener listener : getListenersFor(sessionId, roomType)) { + listener.onEndOfCandidates(); + } + } +} diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverWebRtcTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverWebRtcTest.java new file mode 100644 index 000000000..da85f8493 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverWebRtcTest.java @@ -0,0 +1,366 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.signaling.NCIceCandidate; +import com.nextcloud.talk.models.json.signaling.NCMessagePayload; +import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class SignalingMessageReceiverWebRtcTest { + + private SignalingMessageReceiver signalingMessageReceiver; + + @Before + public void setUp() { + // SignalingMessageReceiver is abstract to prevent direct instantiation without calling the appropriate + // protected methods. + signalingMessageReceiver = new SignalingMessageReceiver() { + }; + } + + @Test + public void testAddWebRtcMessageListenerWithNullListener() { + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener(null, "theSessionId", "theRoomType"); + }); + } + + @Test + public void testAddWebRtcMessageListenerWithNullSessionId() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, null, "theRoomType"); + }); + } + + @Test + public void testAddWebRtcMessageListenerWithNullRoomType() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", null); + }); + } + + @Test + public void testWebRtcMessageOffer() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onOffer("theSdp", null); + } + + @Test + public void testWebRtcMessageOfferWithNick() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onOffer("theSdp", "theNick"); + } + + @Test + public void testWebRtcMessageAnswer() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("answer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("answer"); + messagePayload.setSdp("theSdp"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onAnswer("theSdp", null); + } + + @Test + public void testWebRtcMessageAnswerWithNick() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("answer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("answer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onAnswer("theSdp", "theNick"); + } + + @Test + public void testWebRtcMessageCandidate() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("candidate"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + NCIceCandidate iceCandidate = new NCIceCandidate(); + iceCandidate.setSdpMid("theSdpMid"); + iceCandidate.setSdpMLineIndex(42); + iceCandidate.setCandidate("theSdp"); + messagePayload.setIceCandidate(iceCandidate); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onCandidate("theSdpMid", 42, "theSdp"); + } + + @Test + public void testWebRtcMessageEndOfCandidates() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onEndOfCandidates(); + } + + @Test + public void testWebRtcMessageSeveralListenersSameFrom() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener1 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener2 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener1, "theSessionId", "theRoomType"); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener2, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener1, only()).onEndOfCandidates(); + verify(mockedWebRtcMessageListener2, only()).onEndOfCandidates(); + } + + @Test + public void testWebRtcMessageNotMatchingSessionId() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("notMatchingSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedWebRtcMessageListener); + } + + @Test + public void testWebRtcMessageNotMatchingRoomType() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("notMatchingRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedWebRtcMessageListener); + } + + @Test + public void testWebRtcMessageAfterRemovingListener() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + signalingMessageReceiver.removeListener(mockedWebRtcMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedWebRtcMessageListener); + } + + @Test + public void testWebRtcMessageAfterRemovingSingleListenerOfSeveral() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener1 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener2 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener3 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener1, "theSessionId", "theRoomType"); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener2, "theSessionId", "theRoomType"); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener3, "theSessionId", "theRoomType"); + signalingMessageReceiver.removeListener(mockedWebRtcMessageListener2); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener1, only()).onEndOfCandidates(); + verify(mockedWebRtcMessageListener3, only()).onEndOfCandidates(); + verifyNoInteractions(mockedWebRtcMessageListener2); + } + + @Test + public void testWebRtcMessageAfterAddingListenerAgainForDifferentFrom() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId2", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedWebRtcMessageListener); + + signalingMessage.setFrom("theSessionId2"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener, only()).onEndOfCandidates(); + } + + @Test + public void testAddWebRtcMessageListenerWhenHandlingWebRtcMessage() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener1 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener2 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.addListener(mockedWebRtcMessageListener2, "theSessionId", "theRoomType"); + return null; + }).when(mockedWebRtcMessageListener1).onEndOfCandidates(); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener1, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedWebRtcMessageListener1, only()).onEndOfCandidates(); + verifyNoInteractions(mockedWebRtcMessageListener2); + } + + @Test + public void testRemoveWebRtcMessageListenerWhenHandlingWebRtcMessage() { + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener1 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener2 = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.removeListener(mockedWebRtcMessageListener2); + return null; + }).when(mockedWebRtcMessageListener1).onEndOfCandidates(); + + signalingMessageReceiver.addListener(mockedWebRtcMessageListener1, "theSessionId", "theRoomType"); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener2, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("endOfCandidates"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + InOrder inOrder = inOrder(mockedWebRtcMessageListener1, mockedWebRtcMessageListener2); + + inOrder.verify(mockedWebRtcMessageListener1).onEndOfCandidates(); + inOrder.verify(mockedWebRtcMessageListener2).onEndOfCandidates(); + } +} From 476fb59a08c7c02a1eeddc1d20cab18edbedef3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 17 Oct 2022 10:55:35 +0200 Subject: [PATCH 07/14] Use temporary SignalingMessageReceiver implementation in CallActivity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eventually all signaling related code should be moved to a Signaling class that abstracts the differences between the internal and external signaling servers, including how messages are sent and listened to. In the meantime a temporary SignalingMessageReceiver implementation is added to CallActivity to be able to start using it. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 63 +++++++------------ .../talk/webrtc/PeerConnectionWrapper.java | 16 +++-- 2 files changed, 33 insertions(+), 46 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 33528f640..1949a4608 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -78,7 +78,6 @@ import com.nextcloud.talk.models.json.participants.Participant; import com.nextcloud.talk.models.json.participants.ParticipantsOverall; import com.nextcloud.talk.models.json.signaling.DataChannelMessage; import com.nextcloud.talk.models.json.signaling.DataChannelMessageNick; -import com.nextcloud.talk.models.json.signaling.NCIceCandidate; import com.nextcloud.talk.models.json.signaling.NCMessagePayload; import com.nextcloud.talk.models.json.signaling.NCMessageWrapper; import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; @@ -86,6 +85,7 @@ import com.nextcloud.talk.models.json.signaling.Signaling; import com.nextcloud.talk.models.json.signaling.SignalingOverall; import com.nextcloud.talk.models.json.signaling.settings.IceServer; import com.nextcloud.talk.models.json.signaling.settings.SignalingSettingsOverall; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; import com.nextcloud.talk.ui.dialog.AudioOutputDialog; import com.nextcloud.talk.users.UserManager; import com.nextcloud.talk.utils.ApiUtils; @@ -264,6 +264,8 @@ public class CallActivity extends CallBaseActivity { private SpotlightView spotlightView; + private CallActivitySignalingMessageReceiver signalingMessageReceiver = new CallActivitySignalingMessageReceiver(); + private ExternalSignalingServer externalSignalingServer; private MagicWebSocketInstance webSocketClient; private WebSocketConnectionHelper webSocketConnectionHelper; @@ -1670,44 +1672,12 @@ public class CallActivity extends CallBaseActivity { return; } - PeerConnectionWrapper peerConnectionWrapper = null; - if ("offer".equals(type)) { - peerConnectionWrapper = - getOrCreatePeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(), - ncSignalingMessage.getRoomType(), false); - } else { - peerConnectionWrapper = - getPeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(), - ncSignalingMessage.getRoomType()); + getOrCreatePeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(), + ncSignalingMessage.getRoomType(), false); } - if (peerConnectionWrapper == null) { - return; - } - - String sdp = ncSignalingMessage.getPayload().getSdp(); - String nick = ncSignalingMessage.getPayload().getNick(); - - switch (type) { - case "offer": - peerConnectionWrapper.getWebRtcMessageListener().onOffer(sdp, nick); - break; - case "answer": - peerConnectionWrapper.getWebRtcMessageListener().onAnswer(sdp, nick); - break; - case "candidate": - NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); - peerConnectionWrapper.getWebRtcMessageListener().onCandidate(ncIceCandidate.getSdpMid(), - ncIceCandidate.getSdpMLineIndex(), - ncIceCandidate.getCandidate()); - break; - case "endOfCandidates": - peerConnectionWrapper.getWebRtcMessageListener().onEndOfCandidates(); - break; - default: - break; - } + signalingMessageReceiver.process(ncSignalingMessage); } else { Log.e(TAG, "unexpected RoomType while processing NCSignalingMessage"); } @@ -2010,7 +1980,8 @@ public class CallActivity extends CallBaseActivity { localStream, true, true, - type); + type, + signalingMessageReceiver); } else if (hasMCU) { peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, @@ -2021,7 +1992,8 @@ public class CallActivity extends CallBaseActivity { null, false, true, - type); + type, + signalingMessageReceiver); } else { if (!"screen".equals(type)) { peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, @@ -2032,7 +2004,8 @@ public class CallActivity extends CallBaseActivity { localStream, false, false, - type); + type, + signalingMessageReceiver); } else { peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, iceServers, @@ -2042,7 +2015,8 @@ public class CallActivity extends CallBaseActivity { null, false, false, - type); + type, + signalingMessageReceiver); } } @@ -2653,6 +2627,15 @@ public class CallActivity extends CallBaseActivity { } } + /** + * Temporary implementation of SignalingMessageReceiver until signaling related code is extracted from CallActivity. + */ + private static class CallActivitySignalingMessageReceiver extends SignalingMessageReceiver { + public void process(NCSignalingMessage message) { + processSignalingMessage(message); + } + } + private class MicrophoneButtonTouchListener implements View.OnTouchListener { @SuppressLint("ClickableViewAccessibility") diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index f4fbbeba9..a06df1781 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -37,6 +37,7 @@ import com.nextcloud.talk.events.WebSocketCommunicationEvent; import com.nextcloud.talk.models.json.signaling.DataChannelMessage; import com.nextcloud.talk.models.json.signaling.DataChannelMessageNick; import com.nextcloud.talk.models.json.signaling.NCIceCandidate; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; import org.greenrobot.eventbus.EventBus; import org.webrtc.AudioTrack; @@ -74,6 +75,7 @@ public class PeerConnectionWrapper { private static final String TAG = PeerConnectionWrapper.class.getCanonicalName(); + private final SignalingMessageReceiver signalingMessageReceiver; private final WebRtcMessageListener webRtcMessageListener = new WebRtcMessageListener(); private List iceCandidates = new ArrayList<>(); @@ -98,7 +100,8 @@ public class PeerConnectionWrapper { List iceServerList, MediaConstraints mediaConstraints, String sessionId, String localSession, @Nullable MediaStream localStream, - boolean isMCUPublisher, boolean hasMCU, String videoStreamType) { + boolean isMCUPublisher, boolean hasMCU, String videoStreamType, + SignalingMessageReceiver signalingMessageReceiver) { Objects.requireNonNull(NextcloudTalkApplication.Companion.getSharedApplication()).getComponentApplication().inject(this); @@ -116,6 +119,9 @@ public class PeerConnectionWrapper { configuration.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; peerConnection = peerConnectionFactory.createPeerConnection(configuration, new MagicPeerConnectionObserver()); + this.signalingMessageReceiver = signalingMessageReceiver; + this.signalingMessageReceiver.addListener(webRtcMessageListener, sessionId, videoStreamType); + if (peerConnection != null) { if (this.localStream != null) { List localStreamIds = Collections.singletonList(this.localStream.getId()); @@ -152,6 +158,8 @@ public class PeerConnectionWrapper { } public void removePeerConnection() { + signalingMessageReceiver.removeListener(webRtcMessageListener); + if (dataChannel != null) { dataChannel.dispose(); dataChannel = null; @@ -261,11 +269,7 @@ public class PeerConnectionWrapper { return false; } - public WebRtcMessageListener getWebRtcMessageListener() { - return webRtcMessageListener; - } - - public class WebRtcMessageListener { + private class WebRtcMessageListener implements SignalingMessageReceiver.WebRtcMessageListener { public void onOffer(String sdp, String nick) { onOfferOrAnswer("offer", sdp, nick); From d42fe61e89fa4ffe8581079773cba6ffd1e1611a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 18 Oct 2022 01:16:09 +0200 Subject: [PATCH 08/14] Add listener for offer messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlike the WebRtcMessageListener, which is bound to a specific peer connection, an OfferMessageListener listens to all offer messages, no matter which peer connection they are bound to. This can be used, for example, to create a new peer connection when a remote offer for which there is no previous connection is received. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 16 +- .../talk/signaling/OfferMessageNotifier.java | 53 ++++ .../signaling/SignalingMessageReceiver.java | 43 ++++ .../SignalingMessageReceiverOfferTest.java | 231 ++++++++++++++++++ .../SignalingMessageReceiverTest.java | 135 ++++++++++ 5 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/signaling/OfferMessageNotifier.java create mode 100644 app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java create mode 100644 app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverTest.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 1949a4608..8f31f0a49 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -266,6 +266,13 @@ public class CallActivity extends CallBaseActivity { private CallActivitySignalingMessageReceiver signalingMessageReceiver = new CallActivitySignalingMessageReceiver(); + private SignalingMessageReceiver.OfferMessageListener offerMessageListener = new SignalingMessageReceiver.OfferMessageListener() { + @Override + public void onOffer(String sessionId, String roomType, String sdp, String nick) { + getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, roomType, false); + } + }; + private ExternalSignalingServer externalSignalingServer; private MagicWebSocketInstance webSocketClient; private WebSocketConnectionHelper webSocketConnectionHelper; @@ -522,6 +529,8 @@ public class CallActivity extends CallBaseActivity { sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true")); sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); + signalingMessageReceiver.addListener(offerMessageListener); + if (!isVoiceOnlyCall) { cameraInitialization(); } @@ -1206,6 +1215,8 @@ public class CallActivity extends CallBaseActivity { @Override public void onDestroy() { + signalingMessageReceiver.removeListener(offerMessageListener); + if (localStream != null) { localStream.dispose(); localStream = null; @@ -1672,11 +1683,6 @@ public class CallActivity extends CallBaseActivity { return; } - if ("offer".equals(type)) { - getOrCreatePeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(), - ncSignalingMessage.getRoomType(), false); - } - signalingMessageReceiver.process(ncSignalingMessage); } else { Log.e(TAG, "unexpected RoomType while processing NCSignalingMessage"); diff --git a/app/src/main/java/com/nextcloud/talk/signaling/OfferMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/signaling/OfferMessageNotifier.java new file mode 100644 index 000000000..e0c3842cd --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/signaling/OfferMessageNotifier.java @@ -0,0 +1,53 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Helper class to register and notify OfferMessageListeners. + * + * This class is only meant for internal use by SignalingMessageReceiver; listeners must register themselves against + * a SignalingMessageReceiver rather than against an OfferMessageNotifier. + */ +class OfferMessageNotifier { + + private final Set offerMessageListeners = new LinkedHashSet<>(); + + public synchronized void addListener(SignalingMessageReceiver.OfferMessageListener listener) { + if (listener == null) { + throw new IllegalArgumentException("OfferMessageListener can not be null"); + } + + offerMessageListeners.add(listener); + } + + public synchronized void removeListener(SignalingMessageReceiver.OfferMessageListener listener) { + offerMessageListeners.remove(listener); + } + + public synchronized void notifyOffer(String sessionId, String roomType, String sdp, String nick) { + for (SignalingMessageReceiver.OfferMessageListener listener : new ArrayList<>(offerMessageListeners)) { + listener.onOffer(sessionId, roomType, sdp, nick); + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java index f798ba9b8..439448ee2 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java +++ b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java @@ -26,6 +26,14 @@ import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; /** * Hub to register listeners for signaling messages of different kinds. * + * In general, if a listener is added while an event is being handled the new listener will not receive that event. + * An exception to that is adding a WebRtcMessageListener when handling an offer in an OfferMessageListener; in that + * case the "onOffer()" method of the WebRtcMessageListener will be called for that same offer. + * + * Similarly, if a listener is removed while an event is being handled the removed listener will still receive that + * event. Again the exception is removing a WebRtcMessageListener when handling an offer in an OfferMessageListener; in + * that case the "onOffer()" method of the WebRtcMessageListener will not be called for that offer. + * * Adding and removing listeners, as well as notifying them is internally synchronized. This should be kept in mind * if listeners are added or removed when handling an event to prevent deadlocks (nevertheless, just adding or * removing a listener in the same thread handling the event is fine, and in most cases it will be fine too if done @@ -36,6 +44,19 @@ import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; */ public abstract class SignalingMessageReceiver { + /** + * Listener for WebRTC offers. + * + * Unlike the WebRtcMessageListener, which is bound to a specific peer connection, an OfferMessageListener listens + * to all offer messages, no matter which peer connection they are bound to. This can be used, for example, to + * create a new peer connection when a remote offer for which there is no previous connection is received. + * + * When an offer is received all OfferMessageListeners are notified before any WebRtcMessageListener is notified. + */ + public interface OfferMessageListener { + void onOffer(String sessionId, String roomType, String sdp, String nick); + } + /** * Listener for WebRTC messages. * @@ -49,8 +70,25 @@ public abstract class SignalingMessageReceiver { void onEndOfCandidates(); } + private final OfferMessageNotifier offerMessageNotifier = new OfferMessageNotifier(); + private final WebRtcMessageNotifier webRtcMessageNotifier = new WebRtcMessageNotifier(); + /** + * Adds a listener for all offer messages. + * + * A listener is expected to be added only once. If the same listener is added again it will be notified just once. + * + * @param listener the OfferMessageListener + */ + public void addListener(OfferMessageListener listener) { + offerMessageNotifier.addListener(listener); + } + + public void removeListener(OfferMessageListener listener) { + offerMessageNotifier.removeListener(listener); + } + /** * Adds a listener for WebRTC messages from the given session ID and room type. * @@ -126,6 +164,11 @@ public abstract class SignalingMessageReceiver { String sdp = payload.getSdp(); String nick = payload.getNick(); + // If "processSignalingMessage" is called with two offers from two different threads it is possible, + // although extremely unlikely, that the WebRtcMessageListeners for the second offer are notified before the + // WebRtcMessageListeners for the first offer. This should not be a problem, though, so for simplicity + // the statements are not synchronized. + offerMessageNotifier.notifyOffer(sessionId, roomType, sdp, nick); webRtcMessageNotifier.notifyOffer(sessionId, roomType, sdp, nick); return; diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java new file mode 100644 index 000000000..3491b5cbb --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java @@ -0,0 +1,231 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.signaling.NCMessagePayload; +import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class SignalingMessageReceiverOfferTest { + + private SignalingMessageReceiver signalingMessageReceiver; + + @Before + public void setUp() { + // SignalingMessageReceiver is abstract to prevent direct instantiation without calling the appropriate + // protected methods. + signalingMessageReceiver = new SignalingMessageReceiver() { + }; + } + + @Test + public void testAddOfferMessageListenerWithNullListener() { + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener(null); + }); + } + + @Test + public void testOfferMessage() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedOfferMessageListener, only()).onOffer("theSessionId", "theRoomType", "theSdp", null); + } + + @Test + public void testOfferMessageWithNick() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedOfferMessageListener, only()).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + } + + @Test + public void testOfferMessageAfterRemovingListener() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + signalingMessageReceiver.removeListener(mockedOfferMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedOfferMessageListener); + } + + @Test + public void testOfferMessageAfterRemovingSingleListenerOfSeveral() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener1 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener2 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener3 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + signalingMessageReceiver.addListener(mockedOfferMessageListener1); + signalingMessageReceiver.addListener(mockedOfferMessageListener2); + signalingMessageReceiver.addListener(mockedOfferMessageListener3); + signalingMessageReceiver.removeListener(mockedOfferMessageListener2); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedOfferMessageListener1, only()).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + verify(mockedOfferMessageListener3, only()).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + verifyNoInteractions(mockedOfferMessageListener2); + } + + @Test + public void testOfferMessageAfterAddingListenerAgain() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + signalingMessageReceiver.addListener(mockedOfferMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedOfferMessageListener, only()).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + } + + @Test + public void testAddOfferMessageListenerWhenHandlingOffer() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener1 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener2 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.addListener(mockedOfferMessageListener2); + return null; + }).when(mockedOfferMessageListener1).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + + signalingMessageReceiver.addListener(mockedOfferMessageListener1); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedOfferMessageListener1, only()).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + verifyNoInteractions(mockedOfferMessageListener2); + } + + @Test + public void testRemoveOfferMessageListenerWhenHandlingOffer() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener1 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener2 = + mock(SignalingMessageReceiver.OfferMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.removeListener(mockedOfferMessageListener2); + return null; + }).when(mockedOfferMessageListener1).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + + signalingMessageReceiver.addListener(mockedOfferMessageListener1); + signalingMessageReceiver.addListener(mockedOfferMessageListener2); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + InOrder inOrder = inOrder(mockedOfferMessageListener1, mockedOfferMessageListener2); + + inOrder.verify(mockedOfferMessageListener1).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + inOrder.verify(mockedOfferMessageListener2).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + } +} diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverTest.java new file mode 100644 index 000000000..bf00a31c9 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverTest.java @@ -0,0 +1,135 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.signaling.NCMessagePayload; +import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class SignalingMessageReceiverTest { + + private SignalingMessageReceiver signalingMessageReceiver; + + @Before + public void setUp() { + // SignalingMessageReceiver is abstract to prevent direct instantiation without calling the appropriate + // protected methods. + signalingMessageReceiver = new SignalingMessageReceiver() { + }; + } + + @Test + public void testOfferWithOfferAndWebRtcMessageListeners() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + InOrder inOrder = inOrder(mockedOfferMessageListener, mockedWebRtcMessageListener); + + inOrder.verify(mockedOfferMessageListener).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + inOrder.verify(mockedWebRtcMessageListener).onOffer("theSdp", "theNick"); + } + + @Test + public void testAddWebRtcMessageListenerWhenHandlingOffer() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + return null; + }).when(mockedOfferMessageListener).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + InOrder inOrder = inOrder(mockedOfferMessageListener, mockedWebRtcMessageListener); + + inOrder.verify(mockedOfferMessageListener).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + inOrder.verify(mockedWebRtcMessageListener).onOffer("theSdp", "theNick"); + } + + @Test + public void testRemoveWebRtcMessageListenerWhenHandlingOffer() { + SignalingMessageReceiver.OfferMessageListener mockedOfferMessageListener = + mock(SignalingMessageReceiver.OfferMessageListener.class); + SignalingMessageReceiver.WebRtcMessageListener mockedWebRtcMessageListener = + mock(SignalingMessageReceiver.WebRtcMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.removeListener(mockedWebRtcMessageListener); + return null; + }).when(mockedOfferMessageListener).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + + signalingMessageReceiver.addListener(mockedOfferMessageListener); + signalingMessageReceiver.addListener(mockedWebRtcMessageListener, "theSessionId", "theRoomType"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("offer"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("offer"); + messagePayload.setSdp("theSdp"); + messagePayload.setNick("theNick"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedOfferMessageListener, only()).onOffer("theSessionId", "theRoomType", "theSdp", "theNick"); + verifyNoInteractions(mockedWebRtcMessageListener); + } +} From bda7d2719ba46936f67ba753d4a8e7a37becb512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Wed, 19 Oct 2022 17:27:05 +0200 Subject: [PATCH 09/14] Add listener for call participant messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although "unshareScreen" is technically bound to a specific peer connection it is instead treated as a general message on the call participant. Nevertheless, call participant messages will make possible (at a later point) to listen to events like "raise hand" or "mute" (which, again, could be technically bound to a specific peer connection, but at least for now are treated as a general message on the call participant). Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 39 ++- .../CallParticipantMessageNotifier.java | 95 +++++++ .../signaling/SignalingMessageReceiver.java | 69 ++++++ ...ingMessageReceiverCallParticipantTest.java | 233 ++++++++++++++++++ 4 files changed, 428 insertions(+), 8 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java create mode 100644 app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 8f31f0a49..dbc942517 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -266,6 +266,9 @@ public class CallActivity extends CallBaseActivity { private CallActivitySignalingMessageReceiver signalingMessageReceiver = new CallActivitySignalingMessageReceiver(); + private Map callParticipantMessageListeners = + new HashMap<>(); + private SignalingMessageReceiver.OfferMessageListener offerMessageListener = new SignalingMessageReceiver.OfferMessageListener() { @Override public void onOffer(String sessionId, String roomType, String sdp, String nick) { @@ -1675,14 +1678,6 @@ public class CallActivity extends CallBaseActivity { private void processMessage(NCSignalingMessage ncSignalingMessage) { if ("video".equals(ncSignalingMessage.getRoomType()) || "screen".equals(ncSignalingMessage.getRoomType())) { - String type = ncSignalingMessage.getType(); - - if ("unshareScreen".equals(type)) { - endPeerConnection(ncSignalingMessage.getFrom(), true); - - return; - } - signalingMessageReceiver.process(ncSignalingMessage); } else { Log.e(TAG, "unexpected RoomType while processing NCSignalingMessage"); @@ -2028,6 +2023,15 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapperList.add(peerConnectionWrapper); + // Currently there is no separation between call participants and peer connections, so any video peer + // connection (except the own publisher connection) is treated as a call participant. + if (!publisher && "video".equals(type)) { + SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = + new CallActivityCallParticipantMessageListener(sessionId); + callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); + signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); + } + if (publisher) { startSendingNick(); } @@ -2060,6 +2064,11 @@ public class CallActivity extends CallBaseActivity { } } } + + if (!justScreen) { + SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); + signalingMessageReceiver.removeListener(listener); + } } private void removeMediaStream(String sessionId, String videoStreamType) { @@ -2642,6 +2651,20 @@ public class CallActivity extends CallBaseActivity { } } + private class CallActivityCallParticipantMessageListener implements SignalingMessageReceiver.CallParticipantMessageListener { + + private final String sessionId; + + public CallActivityCallParticipantMessageListener(String sessionId) { + this.sessionId = sessionId; + } + + @Override + public void onUnshareScreen() { + endPeerConnection(sessionId, true); + } + } + private class MicrophoneButtonTouchListener implements View.OnTouchListener { @SuppressLint("ClickableViewAccessibility") diff --git a/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java new file mode 100644 index 000000000..f06e72629 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java @@ -0,0 +1,95 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Helper class to register and notify CallParticipantMessageListeners. + * + * This class is only meant for internal use by SignalingMessageReceiver; listeners must register themselves against + * a SignalingMessageReceiver rather than against a CallParticipantMessageNotifier. + */ +class CallParticipantMessageNotifier { + + /** + * Helper class to associate a CallParticipantMessageListener with a session ID. + */ + private static class CallParticipantMessageListenerFrom { + public final SignalingMessageReceiver.CallParticipantMessageListener listener; + public final String sessionId; + + private CallParticipantMessageListenerFrom(SignalingMessageReceiver.CallParticipantMessageListener listener, + String sessionId) { + this.listener = listener; + this.sessionId = sessionId; + } + } + + private final List callParticipantMessageListenersFrom = new ArrayList<>(); + + public synchronized void addListener(SignalingMessageReceiver.CallParticipantMessageListener listener, String sessionId) { + if (listener == null) { + throw new IllegalArgumentException("CallParticipantMessageListener can not be null"); + } + + if (sessionId == null) { + throw new IllegalArgumentException("sessionId can not be null"); + } + + removeListener(listener); + + callParticipantMessageListenersFrom.add(new CallParticipantMessageListenerFrom(listener, sessionId)); + } + + public synchronized void removeListener(SignalingMessageReceiver.CallParticipantMessageListener listener) { + Iterator it = callParticipantMessageListenersFrom.iterator(); + while (it.hasNext()) { + CallParticipantMessageListenerFrom listenerFrom = it.next(); + + if (listenerFrom.listener == listener) { + it.remove(); + + return; + } + } + } + + private List getListenersFor(String sessionId) { + List callParticipantMessageListeners = + new ArrayList<>(callParticipantMessageListenersFrom.size()); + + for (CallParticipantMessageListenerFrom listenerFrom : callParticipantMessageListenersFrom) { + if (listenerFrom.sessionId.equals(sessionId)) { + callParticipantMessageListeners.add(listenerFrom.listener); + } + } + + return callParticipantMessageListeners; + } + + public synchronized void notifyUnshareScreen(String sessionId) { + for (SignalingMessageReceiver.CallParticipantMessageListener listener : getListenersFor(sessionId)) { + listener.onUnshareScreen(); + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java index 439448ee2..6d1a90d45 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java +++ b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java @@ -44,6 +44,19 @@ import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; */ public abstract class SignalingMessageReceiver { + /** + * Listener for call participant messages. + * + * The messages are bound to a specific call participant (or, rather, session), so each listener is expected to + * handle messages only for a single call participant. + * + * Although "unshareScreen" is technically bound to a specific peer connection it is instead treated as a general + * message on the call participant. + */ + public interface CallParticipantMessageListener { + void onUnshareScreen(); + } + /** * Listener for WebRTC offers. * @@ -70,10 +83,29 @@ public abstract class SignalingMessageReceiver { void onEndOfCandidates(); } + private final CallParticipantMessageNotifier callParticipantMessageNotifier = new CallParticipantMessageNotifier(); + private final OfferMessageNotifier offerMessageNotifier = new OfferMessageNotifier(); private final WebRtcMessageNotifier webRtcMessageNotifier = new WebRtcMessageNotifier(); + /** + * Adds a listener for call participant messages. + * + * A listener is expected to be added only once. If the same listener is added again it will no longer be notified + * for the messages from the previous session ID. + * + * @param listener the CallParticipantMessageListener + * @param sessionId the ID of the session that messages come from + */ + public void addListener(CallParticipantMessageListener listener, String sessionId) { + callParticipantMessageNotifier.addListener(listener, sessionId); + } + + public void removeListener(CallParticipantMessageListener listener) { + callParticipantMessageNotifier.removeListener(listener); + } + /** * Adds a listener for all offer messages. * @@ -116,6 +148,43 @@ public abstract class SignalingMessageReceiver { String sessionId = signalingMessage.getFrom(); String roomType = signalingMessage.getRoomType(); + // "unshareScreen" messages are directly sent to the screen peer connection when the internal signaling + // server is used, and to the room when the external signaling server is used. However, the (relevant) data + // of the received message ("from" and "type") is the same in both cases. + if ("unshareScreen".equals(type)) { + // Message schema (external signaling server): + // { + // "type": "message", + // "message": { + // "sender": { + // ... + // }, + // "data": { + // "roomType": "screen", + // "type": "unshareScreen", + // "from": #STRING#, + // }, + // }, + // } + // + // Message schema (internal signaling server): + // { + // "type": "message", + // "data": { + // "to": #STRING#, + // "sid": #STRING#, + // "broadcaster": #STRING#, + // "roomType": "screen", + // "type": "unshareScreen", + // "from": #STRING#, + // }, + // } + + callParticipantMessageNotifier.notifyUnshareScreen(sessionId); + + return; + } + if ("offer".equals(type)) { // Message schema (external signaling server): // { diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java new file mode 100644 index 000000000..01963682e --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java @@ -0,0 +1,233 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class SignalingMessageReceiverCallParticipantTest { + + private SignalingMessageReceiver signalingMessageReceiver; + + @Before + public void setUp() { + // SignalingMessageReceiver is abstract to prevent direct instantiation without calling the appropriate + // protected methods. + signalingMessageReceiver = new SignalingMessageReceiver() { + }; + } + + @Test + public void testAddCallParticipantMessageListenerWithNullListener() { + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener(null, "theSessionId"); + }); + } + + @Test + public void testAddCallParticipantMessageListenerWithNullSessionId() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, null); + }); + } + + @Test + public void testCallParticipantMessageUnshareScreen() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedCallParticipantMessageListener, only()).onUnshareScreen(); + } + + @Test + public void testCallParticipantMessageSeveralListenersSameFrom() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener1 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener2 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener1, "theSessionId"); + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener2, "theSessionId"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedCallParticipantMessageListener1, only()).onUnshareScreen(); + verify(mockedCallParticipantMessageListener2, only()).onUnshareScreen(); + } + + @Test + public void testCallParticipantMessageNotMatchingSessionId() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("notMatchingSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedCallParticipantMessageListener); + } + + @Test + public void testCallParticipantMessageAfterRemovingListener() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId"); + signalingMessageReceiver.removeListener(mockedCallParticipantMessageListener); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedCallParticipantMessageListener); + } + + @Test + public void testCallParticipantMessageAfterRemovingSingleListenerOfSeveral() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener1 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener2 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener3 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener1, "theSessionId"); + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener2, "theSessionId"); + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener3, "theSessionId"); + signalingMessageReceiver.removeListener(mockedCallParticipantMessageListener2); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedCallParticipantMessageListener1, only()).onUnshareScreen(); + verify(mockedCallParticipantMessageListener3, only()).onUnshareScreen(); + verifyNoInteractions(mockedCallParticipantMessageListener2); + } + + @Test + public void testCallParticipantMessageAfterAddingListenerAgainForDifferentFrom() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId"); + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId2"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verifyNoInteractions(mockedCallParticipantMessageListener); + + signalingMessage.setFrom("theSessionId2"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedCallParticipantMessageListener, only()).onUnshareScreen(); + } + + @Test + public void testAddCallParticipantMessageListenerWhenHandlingCallParticipantMessage() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener1 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener2 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener2, "theSessionId"); + return null; + }).when(mockedCallParticipantMessageListener1).onUnshareScreen(); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener1, "theSessionId"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedCallParticipantMessageListener1, only()).onUnshareScreen(); + verifyNoInteractions(mockedCallParticipantMessageListener2); + } + + @Test + public void testRemoveCallParticipantMessageListenerWhenHandlingCallParticipantMessage() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener1 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener2 = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.removeListener(mockedCallParticipantMessageListener2); + return null; + }).when(mockedCallParticipantMessageListener1).onUnshareScreen(); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener1, "theSessionId"); + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener2, "theSessionId"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("unshareScreen"); + signalingMessage.setRoomType("theRoomType"); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + InOrder inOrder = inOrder(mockedCallParticipantMessageListener1, mockedCallParticipantMessageListener2); + + inOrder.verify(mockedCallParticipantMessageListener1).onUnshareScreen(); + inOrder.verify(mockedCallParticipantMessageListener2).onUnshareScreen(); + } +} From 9df56dccdae037c97f59df8f3dcba71a22dcd26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Wed, 19 Oct 2022 18:44:22 +0200 Subject: [PATCH 10/14] Process signaling message directly in the SignalingMessageReceiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will no longer log an error if the room type of the received message is neither "video" nor "screen". However, that should never happen, and it would be useful only while debugging, so it is fine to lose that. Note that the check is not added to SignalingMessageReceiver itself to keep it as generic as possible (and due to the low value of adding it as explained above). Nevertheless, if needed in the future it would be possible to add a special listener that receives raw messages in order to validate them and log the errors, if any. Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/activities/CallActivity.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index dbc942517..baa99b302 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1624,7 +1624,7 @@ public class CallActivity extends CallBaseActivity { break; case "signalingMessage": Log.d(TAG, "onMessageEvent 'signalingMessage'"); - processMessage((NCSignalingMessage) webSocketClient.getJobWithId( + signalingMessageReceiver.process((NCSignalingMessage) webSocketClient.getJobWithId( Integer.valueOf(webSocketCommunicationEvent.getHashMap().get("jobId")))); break; case "peerReadyForRequestingOffer": @@ -1670,17 +1670,9 @@ public class CallActivity extends CallBaseActivity { } else if ("message".equals(messageType)) { NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(), NCSignalingMessage.class); - processMessage(ncSignalingMessage); - } else { - Log.e(TAG, "unexpected message type when receiving signaling message"); - } - } - - private void processMessage(NCSignalingMessage ncSignalingMessage) { - if ("video".equals(ncSignalingMessage.getRoomType()) || "screen".equals(ncSignalingMessage.getRoomType())) { signalingMessageReceiver.process(ncSignalingMessage); } else { - Log.e(TAG, "unexpected RoomType while processing NCSignalingMessage"); + Log.e(TAG, "unexpected message type when receiving signaling message"); } } From c8e77c3d3b55fe1b3c9017f9821ace1f360b990d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 20 Oct 2022 00:21:05 +0200 Subject: [PATCH 11/14] Split message receiver for internal and external signaling servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that the thread used to handle and notify messages from the external signaling server does not change; the EventBus subscriber mode was "BACKGROUND", but as the message was posted from a WebSocket handler, which runs in a worker thread rather than in the main thread, the subscriber was executed in the same thread as the poster. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 22 +++++++++------- .../talk/webrtc/MagicWebSocketInstance.java | 26 +++++++++++++++---- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index baa99b302..24078e5d2 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -264,7 +264,8 @@ public class CallActivity extends CallBaseActivity { private SpotlightView spotlightView; - private CallActivitySignalingMessageReceiver signalingMessageReceiver = new CallActivitySignalingMessageReceiver(); + private InternalSignalingMessageReceiver internalSignalingMessageReceiver = new InternalSignalingMessageReceiver(); + private SignalingMessageReceiver signalingMessageReceiver; private Map callParticipantMessageListeners = new HashMap<>(); @@ -532,8 +533,6 @@ public class CallActivity extends CallBaseActivity { sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true")); sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); - signalingMessageReceiver.addListener(offerMessageListener); - if (!isVoiceOnlyCall) { cameraInitialization(); } @@ -1350,6 +1349,8 @@ public class CallActivity extends CallBaseActivity { if (hasExternalSignalingServer) { setupAndInitiateWebSocketsConnection(); } else { + signalingMessageReceiver = internalSignalingMessageReceiver; + signalingMessageReceiver.addListener(offerMessageListener); joinRoomAndCall(); } } @@ -1548,6 +1549,10 @@ public class CallActivity extends CallBaseActivity { externalSignalingServer.getExternalSignalingServer(), conversationUser, externalSignalingServer.getExternalSignalingTicket(), TextUtils.isEmpty(credentials)); + // Although setupAndInitiateWebSocketsConnection could be called several times the web socket is + // initialized just once, so the message receiver is also initialized just once. + signalingMessageReceiver = webSocketClient.getSignalingMessageReceiver(); + signalingMessageReceiver.addListener(offerMessageListener); } else { if (webSocketClient.isConnected() && currentCallStatus == CallStatus.PUBLISHER_FAILED) { webSocketClient.restartWebSocket(); @@ -1622,11 +1627,6 @@ public class CallActivity extends CallBaseActivity { } break; - case "signalingMessage": - Log.d(TAG, "onMessageEvent 'signalingMessage'"); - signalingMessageReceiver.process((NCSignalingMessage) webSocketClient.getJobWithId( - Integer.valueOf(webSocketCommunicationEvent.getHashMap().get("jobId")))); - break; case "peerReadyForRequestingOffer": Log.d(TAG, "onMessageEvent 'peerReadyForRequestingOffer'"); webSocketClient.requestOfferForSessionIdWithType( @@ -1670,7 +1670,7 @@ public class CallActivity extends CallBaseActivity { } else if ("message".equals(messageType)) { NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(), NCSignalingMessage.class); - signalingMessageReceiver.process(ncSignalingMessage); + internalSignalingMessageReceiver.process(ncSignalingMessage); } else { Log.e(TAG, "unexpected message type when receiving signaling message"); } @@ -2636,8 +2636,10 @@ public class CallActivity extends CallBaseActivity { /** * Temporary implementation of SignalingMessageReceiver until signaling related code is extracted from CallActivity. + * + * All listeners are called in the main thread. */ - private static class CallActivitySignalingMessageReceiver extends SignalingMessageReceiver { + private static class InternalSignalingMessageReceiver extends SignalingMessageReceiver { public void process(NCSignalingMessage message) { processSignalingMessage(message); } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java index bd1a9d429..956a28346 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java @@ -40,6 +40,7 @@ import com.nextcloud.talk.models.json.websocket.ErrorOverallWebSocketMessage; import com.nextcloud.talk.models.json.websocket.EventOverallWebSocketMessage; import com.nextcloud.talk.models.json.websocket.HelloResponseOverallWebSocketMessage; import com.nextcloud.talk.models.json.websocket.JoinedRoomOverallWebSocketMessage; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; import com.nextcloud.talk.utils.MagicMap; import com.nextcloud.talk.utils.bundle.BundleKeys; @@ -109,6 +110,8 @@ public class MagicWebSocketInstance extends WebSocketListener { private List messagesQueue = new ArrayList<>(); + private final ExternalSignalingMessageReceiver signalingMessageReceiver = new ExternalSignalingMessageReceiver(); + MagicWebSocketInstance(User conversationUser, String connectionUrl, String webSocketTicket) { NextcloudTalkApplication.Companion.getSharedApplication().getComponentApplication().inject(this); @@ -326,11 +329,7 @@ public class MagicWebSocketInstance extends WebSocketListener { ncSignalingMessage.setFrom(callOverallWebSocketMessage.getCallWebSocketMessage().getSenderWebSocketMessage().getSessionId()); } - if (!TextUtils.isEmpty(ncSignalingMessage.getFrom())) { - HashMap messageHashMap = new HashMap<>(); - messageHashMap.put(JOB_ID, Integer.toString(magicMap.add(ncSignalingMessage))); - eventBus.post(new WebSocketCommunicationEvent("signalingMessage", messageHashMap)); - } + signalingMessageReceiver.process(ncSignalingMessage); break; case "bye": connected = false; @@ -471,4 +470,21 @@ public class MagicWebSocketInstance extends WebSocketListener { restartWebSocket(); } } + + public SignalingMessageReceiver getSignalingMessageReceiver() { + return signalingMessageReceiver; + } + + /** + * Temporary implementation of SignalingMessageReceiver until signaling related code is extracted to a Signaling + * class. + * + * All listeners are called in the WebSocket reader thread. This thread should be the same as long as the + * WebSocket stays connected, but it may change whenever it is connected again. + */ + private static class ExternalSignalingMessageReceiver extends SignalingMessageReceiver { + public void process(NCSignalingMessage message) { + processSignalingMessage(message); + } + } } From e0c676bb35db1e33fde28d6d8a266742e7424f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 21 Oct 2022 19:28:11 +0200 Subject: [PATCH 12/14] Add listener for participant list messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now only the same participant list messages that were already handled are taken into account, but at a later point further messages, like participants joining or leaving the conversation, could be added too. Signed-off-by: Daniel Calviño Sánchez --- .../ParticipantListMessageNotifier.java | 68 +++ .../signaling/SignalingMessageReceiver.java | 267 ++++++++++ .../SignalingMessageReceiverOfferTest.java | 2 +- ...ingMessageReceiverParticipantListTest.java | 466 ++++++++++++++++++ 4 files changed, 802 insertions(+), 1 deletion(-) create mode 100644 app/src/main/java/com/nextcloud/talk/signaling/ParticipantListMessageNotifier.java create mode 100644 app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java diff --git a/app/src/main/java/com/nextcloud/talk/signaling/ParticipantListMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/signaling/ParticipantListMessageNotifier.java new file mode 100644 index 000000000..b37c3f659 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/signaling/ParticipantListMessageNotifier.java @@ -0,0 +1,68 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.participants.Participant; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Helper class to register and notify ParticipantListMessageListeners. + * + * This class is only meant for internal use by SignalingMessageReceiver; listeners must register themselves against + * a SignalingMessageReceiver rather than against a ParticipantListMessageNotifier. + */ +class ParticipantListMessageNotifier { + + private final Set participantListMessageListeners = new LinkedHashSet<>(); + + public synchronized void addListener(SignalingMessageReceiver.ParticipantListMessageListener listener) { + if (listener == null) { + throw new IllegalArgumentException("participantListMessageListeners can not be null"); + } + + participantListMessageListeners.add(listener); + } + + public synchronized void removeListener(SignalingMessageReceiver.ParticipantListMessageListener listener) { + participantListMessageListeners.remove(listener); + } + + public synchronized void notifyUsersInRoom(List participants) { + for (SignalingMessageReceiver.ParticipantListMessageListener listener : new ArrayList<>(participantListMessageListeners)) { + listener.onUsersInRoom(participants); + } + } + + public synchronized void notifyParticipantsUpdate(List participants) { + for (SignalingMessageReceiver.ParticipantListMessageListener listener : new ArrayList<>(participantListMessageListeners)) { + listener.onParticipantsUpdate(participants); + } + } + + public synchronized void notifyAllParticipantsUpdate(long inCall) { + for (SignalingMessageReceiver.ParticipantListMessageListener listener : new ArrayList<>(participantListMessageListeners)) { + listener.onAllParticipantsUpdate(inCall); + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java index 6d1a90d45..161dae555 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java +++ b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java @@ -19,10 +19,16 @@ */ package com.nextcloud.talk.signaling; +import com.nextcloud.talk.models.json.converters.EnumParticipantTypeConverter; +import com.nextcloud.talk.models.json.participants.Participant; import com.nextcloud.talk.models.json.signaling.NCIceCandidate; import com.nextcloud.talk.models.json.signaling.NCMessagePayload; import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + /** * Hub to register listeners for signaling messages of different kinds. * @@ -44,6 +50,74 @@ import com.nextcloud.talk.models.json.signaling.NCSignalingMessage; */ public abstract class SignalingMessageReceiver { + /** + * Listener for participant list messages. + * + * The messages are implicitly bound to the room currently joined in the signaling server; listeners are expected + * to know the current room. + */ + public interface ParticipantListMessageListener { + + /** + * List of all the participants in the room. + * + * This message is received only when the internal signaling server is used. + * + * The message is received periodically, and the participants may not have been modified since the last message. + * + * Only the following participant properties are set: + * - inCall + * - lastPing + * - sessionId + * - userId (if the participant is not a guest) + * + * "participantPermissions" is provided in the message (since Talk 13), but not currently set in the + * participant. "publishingPermissions" was provided instead in Talk 12, but it was not used anywhere, so it is + * ignored. + * + * @param participants all the participants (users and guests) in the room + */ + void onUsersInRoom(List participants); + + /** + * List of all the participants in the call or the room (depending on what triggered the event). + * + * This message is received only when the external signaling server is used. + * + * The message is received when any participant changed, although what changed is not provided and should be + * derived from the difference with previous messages. The list of participants may include only the + * participants in the call (including those that just left it and thus triggered the event) or all the + * participants currently in the room (participants in the room but not currently active, that is, without a + * session, are not included). + * + * Only the following participant properties are set: + * - inCall + * - lastPing + * - sessionId + * - type + * - userId (if the participant is not a guest) + * + * "nextcloudSessionId" is provided in the message (when the "inCall" property of any participant changed), but + * not currently set in the participant. + * + * "participantPermissions" is provided in the message (since Talk 13), but not currently set in the + * participant. "publishingPermissions" was provided instead in Talk 12, but it was not used anywhere, so it is + * ignored. + * + * @param participants all the participants (users and guests) in the room + */ + void onParticipantsUpdate(List participants); + + /** + * Update of the properties of all the participants in the room. + * + * This message is received only when the external signaling server is used. + * + * @param inCall the new value of the inCall property + */ + void onAllParticipantsUpdate(long inCall); + } + /** * Listener for call participant messages. * @@ -83,12 +157,29 @@ public abstract class SignalingMessageReceiver { void onEndOfCandidates(); } + private final ParticipantListMessageNotifier participantListMessageNotifier = new ParticipantListMessageNotifier(); + private final CallParticipantMessageNotifier callParticipantMessageNotifier = new CallParticipantMessageNotifier(); private final OfferMessageNotifier offerMessageNotifier = new OfferMessageNotifier(); private final WebRtcMessageNotifier webRtcMessageNotifier = new WebRtcMessageNotifier(); + /** + * Adds a listener for participant list messages. + * + * A listener is expected to be added only once. If the same listener is added again it will be notified just once. + * + * @param listener the ParticipantListMessageListener + */ + public void addListener(ParticipantListMessageListener listener) { + participantListMessageNotifier.addListener(listener); + } + + public void removeListener(ParticipantListMessageListener listener) { + participantListMessageNotifier.removeListener(listener); + } + /** * Adds a listener for call participant messages. * @@ -139,6 +230,182 @@ public abstract class SignalingMessageReceiver { webRtcMessageNotifier.removeListener(listener); } + protected void processEvent(Map eventMap) { + if (!"update".equals(eventMap.get("type")) || !"participants".equals(eventMap.get("target"))) { + return; + } + + Map updateMap; + try { + updateMap = (Map) eventMap.get("update"); + } catch (RuntimeException e) { + // Broken message, this should not happen. + return; + } + + if (updateMap == null) { + // Broken message, this should not happen. + return; + } + + if (updateMap.get("all") != null && Boolean.parseBoolean(updateMap.get("all").toString())) { + processAllParticipantsUpdate(updateMap); + + return; + } + + if (updateMap.get("users") != null) { + processParticipantsUpdate(updateMap); + + return; + } + } + + private void processAllParticipantsUpdate(Map updateMap) { + // Message schema: + // { + // "type": "event", + // "event": { + // "target": "participants", + // "type": "update", + // "update": { + // "roomid": #STRING#, + // "incall": 0, + // "all": true, + // }, + // }, + // } + + long inCall; + try { + inCall = Long.parseLong(updateMap.get("inCall").toString()); + } catch (RuntimeException e) { + // Broken message, this should not happen. + return; + } + + participantListMessageNotifier.notifyAllParticipantsUpdate(inCall); + } + + private void processParticipantsUpdate(Map updateMap) { + // Message schema: + // { + // "type": "event", + // "event": { + // "target": "participants", + // "type": "update", + // "update": { + // "roomid": #INTEGER#, + // "users": [ + // { + // "inCall": #INTEGER#, + // "lastPing": #INTEGER#, + // "sessionId": #STRING#, + // "participantType": #INTEGER#, + // "userId": #STRING#, // Optional + // "nextcloudSessionId": #STRING#, // Optional + // "participantPermissions": #INTEGER#, // Talk >= 13 + // }, + // ... + // ], + // }, + // }, + // } + // + // Note that "userId" in participants->update comes from the Nextcloud server, so it is "userId"; in other + // messages, like room->join, it comes directly from the external signaling server, so it is "userid" instead. + + List> users; + try { + users = (List>) updateMap.get("users"); + } catch (RuntimeException e) { + // Broken message, this should not happen. + return; + } + + if (users == null) { + // Broken message, this should not happen. + return; + } + + List participants = new ArrayList<>(users.size()); + + for (Map user: users) { + try { + participants.add(getParticipantFromMessageMap(user)); + } catch (RuntimeException e) { + // Broken message, this should not happen. + return; + } + } + + participantListMessageNotifier.notifyParticipantsUpdate(participants); + } + + protected void processUsersInRoom(List> users) { + // Message schema: + // { + // "type": "usersInRoom", + // "data": [ + // { + // "inCall": #INTEGER#, + // "lastPing": #INTEGER#, + // "roomId": #INTEGER#, + // "sessionId": #STRING#, + // "userId": #STRING#, // Always included, although it can be empty + // "participantPermissions": #INTEGER#, // Talk >= 13 + // }, + // ... + // ], + // } + + List participants = new ArrayList<>(users.size()); + + for (Map user: users) { + try { + participants.add(getParticipantFromMessageMap(user)); + } catch (RuntimeException e) { + // Broken message, this should not happen. + return; + } + } + + participantListMessageNotifier.notifyUsersInRoom(participants); + } + + /** + * Creates and initializes a Participant from the data in the given map. + * + * Maps from internal and external signaling server messages can be used. Nevertheless, besides the differences + * between the messages and the optional properties, it is expected that the message is correct and the given data + * is parseable. Broken messages (for example, a string instead of an integer for "inCall" or a missing + * "sessionId") may cause a RuntimeException to be thrown. + * + * @param participantMap the map with the participant data + * @return the Participant + */ + private Participant getParticipantFromMessageMap(Map participantMap) { + Participant participant = new Participant(); + + participant.setInCall(Long.parseLong(participantMap.get("inCall").toString())); + participant.setLastPing(Long.parseLong(participantMap.get("lastPing").toString())); + participant.setSessionId(participantMap.get("sessionId").toString()); + + if (participantMap.get("userId") != null && !participantMap.get("userId").toString().isEmpty()) { + participant.setUserId(participantMap.get("userId").toString()); + } + + // Only in external signaling messages + if (participantMap.get("participantType") != null) { + int participantTypeInt = Integer.parseInt(participantMap.get("participantType").toString()); + + EnumParticipantTypeConverter converter = new EnumParticipantTypeConverter(); + participant.setType(converter.getFromInt(participantTypeInt)); + } + + return participant; + } + protected void processSignalingMessage(NCSignalingMessage signalingMessage) { // Note that in the internal signaling server message "data" is the String representation of a JSON // object, although it is already decoded when used here. diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java index 3491b5cbb..2163b6852 100644 --- a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverOfferTest.java @@ -49,7 +49,7 @@ public class SignalingMessageReceiverOfferTest { @Test public void testAddOfferMessageListenerWithNullListener() { Assert.assertThrows(IllegalArgumentException.class, () -> { - signalingMessageReceiver.addListener(null); + signalingMessageReceiver.addListener((SignalingMessageReceiver.OfferMessageListener) null); }); } diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java new file mode 100644 index 000000000..f3c965071 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverParticipantListTest.java @@ -0,0 +1,466 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.signaling; + +import com.nextcloud.talk.models.json.participants.Participant; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class SignalingMessageReceiverParticipantListTest { + + private SignalingMessageReceiver signalingMessageReceiver; + + @Before + public void setUp() { + // SignalingMessageReceiver is abstract to prevent direct instantiation without calling the appropriate + // protected methods. + signalingMessageReceiver = new SignalingMessageReceiver() { + }; + } + + @Test + public void testAddParticipantListMessageListenerWithNullListener() { + Assert.assertThrows(IllegalArgumentException.class, () -> { + signalingMessageReceiver.addListener((SignalingMessageReceiver.ParticipantListMessageListener) null); + }); + } + + @Test + public void testInternalSignalingParticipantListMessageUsersInRoom() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + + List> users = new ArrayList<>(2); + Map user1 = new HashMap<>(); + user1.put("inCall", 7); + user1.put("lastPing", 4815); + user1.put("roomId", 108); + user1.put("sessionId", "theSessionId1"); + user1.put("userId", "theUserId"); + // If "participantPermissions" is set in any of the participants all the other participants in the message + // would have it too. But for test simplicity, and as it is not relevant for the processing, in this test it + // is included only in one of the participants. + user1.put("participantPermissions", 42); + users.add(user1); + Map user2 = new HashMap<>(); + user2.put("inCall", 0); + user2.put("lastPing", 162342); + user2.put("roomId", 108); + user2.put("sessionId", "theSessionId2"); + user2.put("userId", ""); + users.add(user2); + signalingMessageReceiver.processUsersInRoom(users); + + List expectedParticipantList = new ArrayList<>(); + Participant expectedParticipant1 = new Participant(); + expectedParticipant1.setInCall(Participant.InCallFlags.IN_CALL | Participant.InCallFlags.WITH_AUDIO | Participant.InCallFlags.WITH_VIDEO); + expectedParticipant1.setLastPing(4815); + expectedParticipant1.setSessionId("theSessionId1"); + expectedParticipant1.setUserId("theUserId"); + expectedParticipantList.add(expectedParticipant1); + + Participant expectedParticipant2 = new Participant(); + expectedParticipant2.setInCall(Participant.InCallFlags.DISCONNECTED); + expectedParticipant2.setLastPing(162342); + expectedParticipant2.setSessionId("theSessionId2"); + expectedParticipantList.add(expectedParticipant2); + + verify(mockedParticipantListMessageListener, only()).onUsersInRoom(expectedParticipantList); + } + + @Test + public void testInternalSignalingParticipantListMessageAfterRemovingListener() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + signalingMessageReceiver.removeListener(mockedParticipantListMessageListener); + + List> users = new ArrayList<>(1); + Map user = new HashMap<>(); + user.put("inCall", 0); + user.put("lastPing", 4815); + user.put("roomId", 108); + user.put("sessionId", "theSessionId"); + user.put("userId", ""); + users.add(user); + signalingMessageReceiver.processUsersInRoom(users); + + verifyNoInteractions(mockedParticipantListMessageListener); + } + + @Test + public void testInternalSignalingParticipantListMessageAfterRemovingSingleListenerOfSeveral() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener1 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener2 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener3 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener1); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener2); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener3); + signalingMessageReceiver.removeListener(mockedParticipantListMessageListener2); + + List> users = new ArrayList<>(1); + Map user = new HashMap<>(); + user.put("inCall", 0); + user.put("lastPing", 4815); + user.put("roomId", 108); + user.put("sessionId", "theSessionId"); + user.put("userId", ""); + users.add(user); + signalingMessageReceiver.processUsersInRoom(users); + + List expectedParticipantList = new ArrayList<>(); + Participant expectedParticipant = new Participant(); + expectedParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + expectedParticipant.setLastPing(4815); + expectedParticipant.setSessionId("theSessionId"); + expectedParticipantList.add(expectedParticipant); + + verify(mockedParticipantListMessageListener1, only()).onUsersInRoom(expectedParticipantList); + verify(mockedParticipantListMessageListener3, only()).onUsersInRoom(expectedParticipantList); + verifyNoInteractions(mockedParticipantListMessageListener2); + } + + @Test + public void testInternalSignalingParticipantListMessageAfterAddingListenerAgain() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + + List> users = new ArrayList<>(1); + Map user = new HashMap<>(); + user.put("inCall", 0); + user.put("lastPing", 4815); + user.put("roomId", 108); + user.put("sessionId", "theSessionId"); + user.put("userId", ""); + users.add(user); + signalingMessageReceiver.processUsersInRoom(users); + + List expectedParticipantList = new ArrayList<>(); + Participant expectedParticipant = new Participant(); + expectedParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + expectedParticipant.setLastPing(4815); + expectedParticipant.setSessionId("theSessionId"); + expectedParticipantList.add(expectedParticipant); + + verify(mockedParticipantListMessageListener, only()).onUsersInRoom(expectedParticipantList); + } + + @Test + public void testAddParticipantListMessageListenerWhenHandlingInternalSignalingParticipantListMessage() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener1 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener2 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + List expectedParticipantList = new ArrayList<>(); + Participant expectedParticipant = new Participant(); + expectedParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + expectedParticipant.setLastPing(4815); + expectedParticipant.setSessionId("theSessionId"); + expectedParticipantList.add(expectedParticipant); + + doAnswer((invocation) -> { + signalingMessageReceiver.addListener(mockedParticipantListMessageListener2); + return null; + }).when(mockedParticipantListMessageListener1).onUsersInRoom(expectedParticipantList); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener1); + + List> users = new ArrayList<>(1); + Map user = new HashMap<>(); + user.put("inCall", 0); + user.put("lastPing", 4815); + user.put("roomId", 108); + user.put("sessionId", "theSessionId"); + user.put("userId", ""); + users.add(user); + signalingMessageReceiver.processUsersInRoom(users); + + verify(mockedParticipantListMessageListener1, only()).onUsersInRoom(expectedParticipantList); + verifyNoInteractions(mockedParticipantListMessageListener2); + } + + @Test + public void testRemoveParticipantListMessageListenerWhenHandlingInternalSignalingParticipantListMessage() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener1 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener2 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + List expectedParticipantList = new ArrayList<>(); + Participant expectedParticipant = new Participant(); + expectedParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + expectedParticipant.setLastPing(4815); + expectedParticipant.setSessionId("theSessionId"); + expectedParticipantList.add(expectedParticipant); + + doAnswer((invocation) -> { + signalingMessageReceiver.removeListener(mockedParticipantListMessageListener2); + return null; + }).when(mockedParticipantListMessageListener1).onUsersInRoom(expectedParticipantList); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener1); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener2); + + List> users = new ArrayList<>(1); + Map user = new HashMap<>(); + user.put("inCall", 0); + user.put("lastPing", 4815); + user.put("roomId", 108); + user.put("sessionId", "theSessionId"); + user.put("userId", ""); + users.add(user); + signalingMessageReceiver.processUsersInRoom(users); + + InOrder inOrder = inOrder(mockedParticipantListMessageListener1, mockedParticipantListMessageListener2); + + inOrder.verify(mockedParticipantListMessageListener1).onUsersInRoom(expectedParticipantList); + inOrder.verify(mockedParticipantListMessageListener2).onUsersInRoom(expectedParticipantList); + } + + @Test + public void testExternalSignalingParticipantListMessageParticipantsUpdate() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + Map updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + List> users = new ArrayList<>(2); + Map user1 = new HashMap<>(); + user1.put("inCall", 7); + user1.put("lastPing", 4815); + user1.put("sessionId", "theSessionId1"); + user1.put("participantType", 3); + user1.put("userId", "theUserId"); + // If "nextcloudSessionId" or "participantPermissions" is set in any of the participants all the other + // participants in the message would have them too. But for test simplicity, and as it is not relevant for + // the processing, in this test they are included only in one of the participants. + user1.put("nextcloudSessionId", "theNextcloudSessionId"); + user1.put("participantPermissions", 42); + users.add(user1); + Map user2 = new HashMap<>(); + user2.put("inCall", 0); + user2.put("lastPing", 162342); + user2.put("sessionId", "theSessionId2"); + user2.put("participantType", 4); + users.add(user2); + updateMap.put("users", users); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + List expectedParticipantList = new ArrayList<>(2); + Participant expectedParticipant1 = new Participant(); + expectedParticipant1.setInCall(Participant.InCallFlags.IN_CALL | Participant.InCallFlags.WITH_AUDIO | Participant.InCallFlags.WITH_VIDEO); + expectedParticipant1.setLastPing(4815); + expectedParticipant1.setSessionId("theSessionId1"); + expectedParticipant1.setType(Participant.ParticipantType.USER); + expectedParticipant1.setUserId("theUserId"); + expectedParticipantList.add(expectedParticipant1); + + Participant expectedParticipant2 = new Participant(); + expectedParticipant2.setInCall(Participant.InCallFlags.DISCONNECTED); + expectedParticipant2.setLastPing(162342); + expectedParticipant2.setSessionId("theSessionId2"); + expectedParticipant2.setType(Participant.ParticipantType.GUEST); + expectedParticipantList.add(expectedParticipant2); + + verify(mockedParticipantListMessageListener, only()).onParticipantsUpdate(expectedParticipantList); + } + + @Test + public void testExternalSignalingParticipantListMessageAllParticipantsUpdate() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + Map updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + updateMap.put("all", true); + updateMap.put("inCall", 0); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + verify(mockedParticipantListMessageListener, only()).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + } + + @Test + public void testExternalSignalingParticipantListMessageAfterRemovingListener() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + signalingMessageReceiver.removeListener(mockedParticipantListMessageListener); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + HashMap updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + updateMap.put("all", true); + updateMap.put("inCall", 0); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + verifyNoInteractions(mockedParticipantListMessageListener); + } + + @Test + public void testExternalSignalingParticipantListMessageAfterRemovingSingleListenerOfSeveral() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener1 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener2 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener3 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener1); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener2); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener3); + signalingMessageReceiver.removeListener(mockedParticipantListMessageListener2); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + HashMap updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + updateMap.put("all", true); + updateMap.put("inCall", 0); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + verify(mockedParticipantListMessageListener1, only()).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + verify(mockedParticipantListMessageListener3, only()).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + verifyNoInteractions(mockedParticipantListMessageListener2); + } + + @Test + public void testExternalSignalingParticipantListMessageAfterAddingListenerAgain() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + HashMap updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + updateMap.put("all", true); + updateMap.put("inCall", 0); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + verify(mockedParticipantListMessageListener, only()).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + } + + @Test + public void testAddParticipantListMessageListenerWhenHandlingExternalSignalingParticipantListMessage() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener1 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener2 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.addListener(mockedParticipantListMessageListener2); + return null; + }).when(mockedParticipantListMessageListener1).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener1); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + HashMap updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + updateMap.put("all", true); + updateMap.put("inCall", 0); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + verify(mockedParticipantListMessageListener1, only()).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + verifyNoInteractions(mockedParticipantListMessageListener2); + } + + @Test + public void testRemoveParticipantListMessageListenerWhenHandlingExternalSignalingParticipantListMessage() { + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener1 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + SignalingMessageReceiver.ParticipantListMessageListener mockedParticipantListMessageListener2 = + mock(SignalingMessageReceiver.ParticipantListMessageListener.class); + + doAnswer((invocation) -> { + signalingMessageReceiver.removeListener(mockedParticipantListMessageListener2); + return null; + }).when(mockedParticipantListMessageListener1).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + + signalingMessageReceiver.addListener(mockedParticipantListMessageListener1); + signalingMessageReceiver.addListener(mockedParticipantListMessageListener2); + + Map eventMap = new HashMap<>(); + eventMap.put("type", "update"); + eventMap.put("target", "participants"); + HashMap updateMap = new HashMap<>(); + updateMap.put("roomId", 108); + updateMap.put("all", true); + updateMap.put("inCall", 0); + eventMap.put("update", updateMap); + signalingMessageReceiver.processEvent(eventMap); + + InOrder inOrder = inOrder(mockedParticipantListMessageListener1, mockedParticipantListMessageListener2); + + inOrder.verify(mockedParticipantListMessageListener1).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + inOrder.verify(mockedParticipantListMessageListener2).onAllParticipantsUpdate(Participant.InCallFlags.DISCONNECTED); + } +} From 5e224c5a247a8e67133d753560df4fbce779f7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 22 Oct 2022 02:57:44 +0200 Subject: [PATCH 13/14] Use listener for participant list messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that the thread used to handle the participant list messages from the external signaling server does not change; the EventBus subscriber mode was "BACKGROUND", but as the message was posted from a WebSocket handler, which runs in a worker thread rather than in the main thread, the subscriber was executed in the same thread as the poster. Also note that the removed "userId" remark was not fully accurate; although some external signaling messages do actually use "userid" those currently handled to process the users do not, they always use "userId" (as documented in the SignalingMessageReceiver). Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 82 ++++++++----------- .../talk/webrtc/MagicWebSocketInstance.java | 44 ++-------- 2 files changed, 41 insertions(+), 85 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 24078e5d2..ac2fbda92 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -270,6 +270,27 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { + + @Override + public void onUsersInRoom(List participants) { + processUsersInRoom(participants); + } + + @Override + public void onParticipantsUpdate(List participants) { + processUsersInRoom(participants); + } + + @Override + public void onAllParticipantsUpdate(long inCall) { + if (inCall == Participant.InCallFlags.DISCONNECTED) { + Log.d(TAG, "A moderator ended the call for all."); + hangup(true); + } + } + }; + private SignalingMessageReceiver.OfferMessageListener offerMessageListener = new SignalingMessageReceiver.OfferMessageListener() { @Override public void onOffer(String sessionId, String roomType, String sdp, String nick) { @@ -1217,6 +1238,7 @@ public class CallActivity extends CallBaseActivity { @Override public void onDestroy() { + signalingMessageReceiver.removeListener(participantListMessageListener); signalingMessageReceiver.removeListener(offerMessageListener); if (localStream != null) { @@ -1350,6 +1372,7 @@ public class CallActivity extends CallBaseActivity { setupAndInitiateWebSocketsConnection(); } else { signalingMessageReceiver = internalSignalingMessageReceiver; + signalingMessageReceiver.addListener(participantListMessageListener); signalingMessageReceiver.addListener(offerMessageListener); joinRoomAndCall(); } @@ -1552,6 +1575,7 @@ public class CallActivity extends CallBaseActivity { // Although setupAndInitiateWebSocketsConnection could be called several times the web socket is // initialized just once, so the message receiver is also initialized just once. signalingMessageReceiver = webSocketClient.getSignalingMessageReceiver(); + signalingMessageReceiver.addListener(participantListMessageListener); signalingMessageReceiver.addListener(offerMessageListener); } else { if (webSocketClient.isConnected() && currentCallStatus == CallStatus.PUBLISHER_FAILED) { @@ -1596,37 +1620,6 @@ public class CallActivity extends CallBaseActivity { performCall(); } break; - case PARTICIPANTS_UPDATE: - Log.d(TAG, "onMessageEvent 'participantsUpdate'"); - - // See MagicWebSocketInstance#onMessage in case "participants" how the 'updateParameters' are created - Map updateParameters = webSocketCommunicationEvent.getHashMap(); - - if (updateParameters == null) { - break; - } - - String updateRoomToken = updateParameters.get(ROOM_TOKEN); - String updateAll = updateParameters.get(UPDATE_ALL); - String updateInCall = updateParameters.get(UPDATE_IN_CALL); - String jobId = updateParameters.get(JOB_ID); - - if (roomToken.equals(updateRoomToken)) { - if (updateAll != null && Boolean.parseBoolean(updateAll)) { - if ("0".equals(updateInCall)) { - Log.d(TAG, "Most probably a moderator ended the call for all."); - hangup(true); - } - } else if (jobId != null) { - // In that case a list of users for the room is passed. - processUsersInRoom( - (List>) webSocketClient - .getJobWithId( - Integer.valueOf(jobId))); - } - - } - break; case "peerReadyForRequestingOffer": Log.d(TAG, "onMessageEvent 'peerReadyForRequestingOffer'"); webSocketClient.requestOfferForSessionIdWithType( @@ -1666,7 +1659,7 @@ public class CallActivity extends CallBaseActivity { } if ("usersInRoom".equals(messageType)) { - processUsersInRoom((List>) signaling.getMessageWrapper()); + internalSignalingMessageReceiver.process((List>) signaling.getMessageWrapper()); } else if ("message".equals(messageType)) { NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(), NCSignalingMessage.class); @@ -1781,7 +1774,7 @@ public class CallActivity extends CallBaseActivity { } } - private void processUsersInRoom(List> users) { + private void processUsersInRoom(List participants) { Log.d(TAG, "processUsersInRoom"); List newSessions = new ArrayList<>(); Set oldSessions = new HashSet<>(); @@ -1800,27 +1793,20 @@ public class CallActivity extends CallBaseActivity { boolean isSelfInCall = false; - for (HashMap participant : users) { - long inCallFlag = (long) participant.get("inCall"); - if (!participant.get("sessionId").equals(currentSessionId)) { + for (Participant participant : participants) { + long inCallFlag = participant.getInCall(); + if (!participant.getSessionId().equals(currentSessionId)) { Log.d(TAG, " inCallFlag of participant " - + participant.get("sessionId").toString().substring(0, 4) + + participant.getSessionId().substring(0, 4) + " : " + inCallFlag); boolean isInCall = inCallFlag != 0; if (isInCall) { - newSessions.add(participant.get("sessionId").toString()); + newSessions.add(participant.getSessionId()); } - // The property is "userId" when not using the external signaling server and "userid" when using it. - String userId = null; - if (participant.get("userId") != null) { - userId = participant.get("userId").toString(); - } else if (participant.get("userid") != null) { - userId = participant.get("userid").toString(); - } - userIdsBySessionId.put(participant.get("sessionId").toString(), userId); + userIdsBySessionId.put(participant.getSessionId(), participant.getUserId()); } else { Log.d(TAG, " inCallFlag of currentSessionId: " + inCallFlag); isSelfInCall = inCallFlag != 0; @@ -2640,6 +2626,10 @@ public class CallActivity extends CallBaseActivity { * All listeners are called in the main thread. */ private static class InternalSignalingMessageReceiver extends SignalingMessageReceiver { + public void process(List> users) { + processUsersInRoom(users); + } + public void process(NCSignalingMessage message) { processSignalingMessage(message); } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java index 956a28346..3c6bd9cc2 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java @@ -279,45 +279,7 @@ public class MagicWebSocketInstance extends WebSocketListener { } break; case TARGET_PARTICIPANTS: - if (EVENT_TYPE_UPDATE.equals(eventOverallWebSocketMessage.getEventMap().get(EVENT_TYPE))) { - HashMap refreshChatHashMap = new HashMap<>(); - HashMap updateEventMap = (HashMap) eventOverallWebSocketMessage.getEventMap().get(EVENT_TYPE_UPDATE); - - if (updateEventMap == null) { - break; - } - - if (updateEventMap.containsKey(UPDATE_ROOM_ID)) { - Object updateRoomId = updateEventMap.get(UPDATE_ROOM_ID); - if (updateRoomId != null) { - refreshChatHashMap.put(ROOM_TOKEN, - (String) updateEventMap.get(UPDATE_ROOM_ID)); - } - } - - if (updateEventMap.containsKey(UPDATE_USERS)) { - Object updateUsers = updateEventMap.get(UPDATE_USERS); - if (updateUsers != null) { - refreshChatHashMap.put(JOB_ID, Integer.toString(magicMap.add(updateUsers))); - } - } - - if (updateEventMap.containsKey(UPDATE_IN_CALL)) { - Object inCall = updateEventMap.get(UPDATE_IN_CALL); - if (inCall != null) { - refreshChatHashMap.put(UPDATE_IN_CALL, Long.toString((Long) inCall)); - } - } - - if (updateEventMap.containsKey(UPDATE_ALL)) { - Object updateAll = updateEventMap.get(UPDATE_ALL); - if (updateAll != null) { - refreshChatHashMap.put(UPDATE_ALL, Boolean.toString((Boolean) updateAll)); - } - } - - eventBus.post(new WebSocketCommunicationEvent(PARTICIPANTS_UPDATE, refreshChatHashMap)); - } + signalingMessageReceiver.process(eventOverallWebSocketMessage.getEventMap()); break; } } @@ -483,6 +445,10 @@ public class MagicWebSocketInstance extends WebSocketListener { * WebSocket stays connected, but it may change whenever it is connected again. */ private static class ExternalSignalingMessageReceiver extends SignalingMessageReceiver { + public void process(Map eventMap) { + processEvent(eventMap); + } + public void process(NCSignalingMessage message) { processSignalingMessage(message); } From c4c64df5a6a9b84010f0cbbfc54caa0563b6b9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 22 Oct 2022 03:09:55 +0200 Subject: [PATCH 14/14] Remove no longer needed code after removing EventBus message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 5 --- .../java/com/nextcloud/talk/utils/MagicMap.kt | 31 ------------------- .../com/nextcloud/talk/webrtc/Globals.java | 11 ------- .../talk/webrtc/MagicWebSocketInstance.java | 17 ---------- 4 files changed, 64 deletions(-) delete mode 100644 app/src/main/java/com/nextcloud/talk/utils/MagicMap.kt diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ac2fbda92..a56a9a09e 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -173,11 +173,6 @@ import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_PARTICIPANT_PERMISS import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_ROOM_ID; import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_ROOM_TOKEN; import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_USER_ENTITY; -import static com.nextcloud.talk.webrtc.Globals.JOB_ID; -import static com.nextcloud.talk.webrtc.Globals.PARTICIPANTS_UPDATE; -import static com.nextcloud.talk.webrtc.Globals.ROOM_TOKEN; -import static com.nextcloud.talk.webrtc.Globals.UPDATE_ALL; -import static com.nextcloud.talk.webrtc.Globals.UPDATE_IN_CALL; @AutoInjector(NextcloudTalkApplication.class) public class CallActivity extends CallBaseActivity { diff --git a/app/src/main/java/com/nextcloud/talk/utils/MagicMap.kt b/app/src/main/java/com/nextcloud/talk/utils/MagicMap.kt deleted file mode 100644 index b0ae58039..000000000 --- a/app/src/main/java/com/nextcloud/talk/utils/MagicMap.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Nextcloud Talk application - * - * @author Mario Danic - * Copyright (C) 2017-2018 Mario Danic - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package com.nextcloud.talk.utils - -import java.util.concurrent.ConcurrentHashMap - -class MagicMap : ConcurrentHashMap() { - fun add(element: Any): Int { - val key = System.identityHashCode(element) - super.put(key, element) - return key - } -} diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/Globals.java b/app/src/main/java/com/nextcloud/talk/webrtc/Globals.java index 8d74fb920..5e27542ee 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/Globals.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/Globals.java @@ -2,17 +2,6 @@ package com.nextcloud.talk.webrtc; public class Globals { public static final String ROOM_TOKEN = "roomToken"; - public static final String JOB_ID = "jobId"; - - public static final String PARTICIPANTS_UPDATE = "participantsUpdate"; public static final String TARGET_PARTICIPANTS = "participants"; - - public static final String EVENT_TYPE = "type"; - public static final String EVENT_TYPE_UPDATE = "update"; - - public static final String UPDATE_ALL = "all"; - public static final String UPDATE_IN_CALL = "incall"; - public static final String UPDATE_ROOM_ID = "roomid"; - public static final String UPDATE_USERS = "users"; } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java index 3c6bd9cc2..733c34df2 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java @@ -41,7 +41,6 @@ import com.nextcloud.talk.models.json.websocket.EventOverallWebSocketMessage; import com.nextcloud.talk.models.json.websocket.HelloResponseOverallWebSocketMessage; import com.nextcloud.talk.models.json.websocket.JoinedRoomOverallWebSocketMessage; import com.nextcloud.talk.signaling.SignalingMessageReceiver; -import com.nextcloud.talk.utils.MagicMap; import com.nextcloud.talk.utils.bundle.BundleKeys; import org.greenrobot.eventbus.EventBus; @@ -66,16 +65,8 @@ import okio.ByteString; import static com.nextcloud.talk.models.json.participants.Participant.ActorType.GUESTS; import static com.nextcloud.talk.models.json.participants.Participant.ActorType.USERS; -import static com.nextcloud.talk.webrtc.Globals.EVENT_TYPE; -import static com.nextcloud.talk.webrtc.Globals.EVENT_TYPE_UPDATE; -import static com.nextcloud.talk.webrtc.Globals.JOB_ID; -import static com.nextcloud.talk.webrtc.Globals.PARTICIPANTS_UPDATE; import static com.nextcloud.talk.webrtc.Globals.ROOM_TOKEN; import static com.nextcloud.talk.webrtc.Globals.TARGET_PARTICIPANTS; -import static com.nextcloud.talk.webrtc.Globals.UPDATE_ALL; -import static com.nextcloud.talk.webrtc.Globals.UPDATE_IN_CALL; -import static com.nextcloud.talk.webrtc.Globals.UPDATE_ROOM_ID; -import static com.nextcloud.talk.webrtc.Globals.UPDATE_USERS; @AutoInjector(NextcloudTalkApplication.class) public class MagicWebSocketInstance extends WebSocketListener { @@ -99,7 +90,6 @@ public class MagicWebSocketInstance extends WebSocketListener { private boolean connected; private WebSocketConnectionHelper webSocketConnectionHelper; private WebSocket internalWebSocket; - private MagicMap magicMap; private String connectionUrl; private String currentRoomToken; @@ -120,7 +110,6 @@ public class MagicWebSocketInstance extends WebSocketListener { this.webSocketTicket = webSocketTicket; this.webSocketConnectionHelper = new WebSocketConnectionHelper(); this.usersHashMap = new HashMap<>(); - magicMap = new MagicMap(); connected = false; eventBus.register(this); @@ -368,12 +357,6 @@ public class MagicWebSocketInstance extends WebSocketListener { } } - public Object getJobWithId(Integer id) { - Object copyJob = magicMap.get(id); - magicMap.remove(id); - return copyJob; - } - public void requestOfferForSessionIdWithType(String sessionIdParam, String roomType) { try { String message = LoganSquare.serialize(webSocketConnectionHelper.getAssembledRequestOfferModel(sessionIdParam, roomType));