From dceb4a6d7973bd109b1102f66b4a403325b1e3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 7 Nov 2022 08:56:20 +0100 Subject: [PATCH] Add listener for data channel messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now only the same data channel messages that were already handled are taken into account, but at a later point the missing messages ("speaking" and "stoppedSpeaking") could be added too. Note that the thread used to handle the data channel messages has changed; the EventBus subscriber mode was "MAIN", but as the messages were posted from a DataChannel observer, which run in a worker thread rather than in the main thread, the subscriber was executed in the main thread rather than in the same thread as the poster. Due to this the actions performed by the handler now must be explicitly run in the main thread. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 90 +++++++++++++++---- .../talk/events/PeerConnectionEvent.java | 2 +- .../webrtc/DataChannelMessageNotifier.java | 77 ++++++++++++++++ .../talk/webrtc/PeerConnectionWrapper.java | 52 ++++++++--- 4 files changed, 189 insertions(+), 32 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/webrtc/DataChannelMessageNotifier.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 22fe59dc4..bfcf198d0 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -268,6 +268,8 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); + private Map dataChannelMessageListeners = new HashMap<>(); + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @Override @@ -2007,6 +2009,12 @@ public class CallActivity extends CallBaseActivity { new CallActivityCallParticipantMessageListener(sessionId); callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); + + // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. + PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = + new CallActivityDataChannelMessageListener(sessionId); + dataChannelMessageListeners.put(sessionId, dataChannelMessageListener); + peerConnectionWrapper.addListener(dataChannelMessageListener); } if (!publisher && !hasExternalSignalingServer && offerAnswerNickProviders.get(sessionId) == null) { @@ -2040,6 +2048,10 @@ public class CallActivity extends CallBaseActivity { if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { if (peerConnectionWrapper.getSessionId().equals(sessionId)) { + if (!justScreen && VIDEO_STREAM_TYPE_VIDEO.equals(peerConnectionWrapper.getVideoStreamType())) { + PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = dataChannelMessageListeners.remove(sessionId); + peerConnectionWrapper.removeListener(dataChannelMessageListener); + } String videoStreamType = peerConnectionWrapper.getVideoStreamType(); if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); @@ -2163,24 +2175,6 @@ public class CallActivity extends CallBaseActivity { toggleMedia(enableVideo, true); } } - } else if (peerConnectionEvent.getPeerConnectionEventType() == - PeerConnectionEvent.PeerConnectionEventType.NICK_CHANGE) { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setNick(peerConnectionEvent.getNick()); - participantsAdapter.notifyDataSetChanged(); - } - } else if (peerConnectionEvent.getPeerConnectionEventType() == - PeerConnectionEvent.PeerConnectionEventType.VIDEO_CHANGE) { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(peerConnectionEvent.getChangeValue()); - participantsAdapter.notifyDataSetChanged(); - } - } else if (peerConnectionEvent.getPeerConnectionEventType() == - PeerConnectionEvent.PeerConnectionEventType.AUDIO_CHANGE) { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(peerConnectionEvent.getChangeValue()); - participantsAdapter.notifyDataSetChanged(); - } } else if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED) { setCallState(CallStatus.PUBLISHER_FAILED); @@ -2631,6 +2625,66 @@ public class CallActivity extends CallBaseActivity { } } + private class CallActivityDataChannelMessageListener implements PeerConnectionWrapper.DataChannelMessageListener { + + private final String participantDisplayItemId; + + private CallActivityDataChannelMessageListener(String sessionId) { + // DataChannel messages are sent only in video peers, so the listener only acts on the "video" items. + this.participantDisplayItemId = sessionId + "-video"; + } + + @Override + public void onAudioOn() { + runOnUiThread(() -> { + if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(true); + participantsAdapter.notifyDataSetChanged(); + } + }); + } + + @Override + public void onAudioOff() { + runOnUiThread(() -> { + if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(false); + participantsAdapter.notifyDataSetChanged(); + } + }); + } + + @Override + public void onVideoOn() { + runOnUiThread(() -> { + if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(true); + participantsAdapter.notifyDataSetChanged(); + } + }); + } + + @Override + public void onVideoOff() { + runOnUiThread(() -> { + if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(false); + participantsAdapter.notifyDataSetChanged(); + } + }); + } + + @Override + public void onNickChanged(String nick) { + runOnUiThread(() -> { + if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setNick(nick); + participantsAdapter.notifyDataSetChanged(); + } + }); + } + } + private class InternalSignalingMessageSender implements SignalingMessageSender { @Override diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java index c6722732e..fd10c30ce 100644 --- a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java @@ -120,6 +120,6 @@ public class PeerConnectionEvent { } public enum PeerConnectionEventType { - PEER_CONNECTED, PEER_DISCONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE, PUBLISHER_FAILED + PEER_CONNECTED, PEER_DISCONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, PUBLISHER_FAILED } } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/DataChannelMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/webrtc/DataChannelMessageNotifier.java new file mode 100644 index 000000000..c949cc39f --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/webrtc/DataChannelMessageNotifier.java @@ -0,0 +1,77 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.webrtc; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Helper class to register and notify DataChannelMessageListeners. + * + * This class is only meant for internal use by PeerConnectionWrapper; listeners must register themselves against + * a PeerConnectionWrapper rather than against a DataChannelMessageNotifier. + */ +public class DataChannelMessageNotifier { + + private final Set dataChannelMessageListeners = new LinkedHashSet<>(); + + public synchronized void addListener(PeerConnectionWrapper.DataChannelMessageListener listener) { + if (listener == null) { + throw new IllegalArgumentException("DataChannelMessageListener can not be null"); + } + + dataChannelMessageListeners.add(listener); + } + + public synchronized void removeListener(PeerConnectionWrapper.DataChannelMessageListener listener) { + dataChannelMessageListeners.remove(listener); + } + + public synchronized void notifyAudioOn() { + for (PeerConnectionWrapper.DataChannelMessageListener listener : new ArrayList<>(dataChannelMessageListeners)) { + listener.onAudioOn(); + } + } + + public synchronized void notifyAudioOff() { + for (PeerConnectionWrapper.DataChannelMessageListener listener : new ArrayList<>(dataChannelMessageListeners)) { + listener.onAudioOff(); + } + } + + public synchronized void notifyVideoOn() { + for (PeerConnectionWrapper.DataChannelMessageListener listener : new ArrayList<>(dataChannelMessageListeners)) { + listener.onVideoOn(); + } + } + + public synchronized void notifyVideoOff() { + for (PeerConnectionWrapper.DataChannelMessageListener listener : new ArrayList<>(dataChannelMessageListeners)) { + listener.onVideoOff(); + } + } + + public synchronized void notifyNickChanged(String nick) { + for (PeerConnectionWrapper.DataChannelMessageListener listener : new ArrayList<>(dataChannelMessageListeners)) { + listener.onNickChanged(nick); + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index e3452f7aa..bb5e57540 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -65,12 +65,26 @@ import javax.inject.Inject; import androidx.annotation.Nullable; import autodagger.AutoInjector; -import static java.lang.Boolean.FALSE; -import static java.lang.Boolean.TRUE; - @AutoInjector(NextcloudTalkApplication.class) public class PeerConnectionWrapper { + /** + * Listener for data channel messages. + * + * The messages are bound to a specific peer connection, so each listener is expected to handle messages only for + * a single peer connection. + * + * All methods are called on the so called "signaling" thread of WebRTC, which is an internal thread created by the + * WebRTC library and NOT the same thread where signaling messages are received. + */ + public interface DataChannelMessageListener { + void onAudioOn(); + void onAudioOff(); + void onVideoOn(); + void onVideoOff(); + void onNickChanged(String nick); + } + private static final String TAG = PeerConnectionWrapper.class.getCanonicalName(); private final SignalingMessageReceiver signalingMessageReceiver; @@ -78,6 +92,8 @@ public class PeerConnectionWrapper { private final SignalingMessageSender signalingMessageSender; + private final DataChannelMessageNotifier dataChannelMessageNotifier = new DataChannelMessageNotifier(); + private List iceCandidates = new ArrayList<>(); private PeerConnection peerConnection; private String sessionId; @@ -156,6 +172,21 @@ public class PeerConnectionWrapper { } } + /** + * Adds a listener for data channel messages. + * + * A listener is expected to be added only once. If the same listener is added again it will be notified just once. + * + * @param listener the DataChannelMessageListener + */ + public void addListener(DataChannelMessageListener listener) { + dataChannelMessageNotifier.addListener(listener); + } + + public void removeListener(DataChannelMessageListener listener) { + dataChannelMessageNotifier.removeListener(listener); + } + public String getVideoStreamType() { return videoStreamType; } @@ -339,21 +370,16 @@ public class PeerConnectionWrapper { } if (nick != null) { - EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .NICK_CHANGE, sessionId, nick, null, videoStreamType)); + dataChannelMessageNotifier.notifyNickChanged(nick); } } else if ("audioOn".equals(dataChannelMessage.getType())) { - EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .AUDIO_CHANGE, sessionId, null, TRUE, videoStreamType)); + dataChannelMessageNotifier.notifyAudioOn(); } else if ("audioOff".equals(dataChannelMessage.getType())) { - EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .AUDIO_CHANGE, sessionId, null, FALSE, videoStreamType)); + dataChannelMessageNotifier.notifyAudioOff(); } else if ("videoOn".equals(dataChannelMessage.getType())) { - EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .VIDEO_CHANGE, sessionId, null, TRUE, videoStreamType)); + dataChannelMessageNotifier.notifyVideoOn(); } else if ("videoOff".equals(dataChannelMessage.getType())) { - EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .VIDEO_CHANGE, sessionId, null, FALSE, videoStreamType)); + dataChannelMessageNotifier.notifyVideoOff(); } } }