From 0a54fd612729d991d72774e25c4d7aa67e51a465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 20 Apr 2023 18:12:53 +0200 Subject: [PATCH 1/3] Add listener for "reaction" signaling message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 4 ++ .../nextcloud/talk/call/CallParticipant.java | 4 ++ .../models/json/signaling/NCMessagePayload.kt | 6 ++- .../CallParticipantMessageNotifier.java | 6 +++ .../signaling/SignalingMessageReceiver.java | 52 +++++++++++++++++++ ...ingMessageReceiverCallParticipantTest.java | 20 +++++++ 6 files changed, 90 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index abd5a7c8c..2791e23f4 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2805,6 +2805,10 @@ public class CallActivity extends CallBaseActivity { } } + @Override + public void onReaction(String reaction) { + } + @Override public void onUnshareScreen() { endPeerConnection(sessionId, "screen"); diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java index d3e202a0c..e0f4df295 100644 --- a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java @@ -40,6 +40,10 @@ public class CallParticipant { callParticipantModel.setRaisedHand(state, timestamp); } + @Override + public void onReaction(String reaction) { + } + @Override public void onUnshareScreen() { } diff --git a/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCMessagePayload.kt b/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCMessagePayload.kt index ce078f11e..1f478b9fc 100644 --- a/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCMessagePayload.kt +++ b/app/src/main/java/com/nextcloud/talk/models/json/signaling/NCMessagePayload.kt @@ -42,8 +42,10 @@ data class NCMessagePayload( @JsonField(name = ["state"]) var state: Boolean? = null, @JsonField(name = ["timestamp"]) - var timestamp: Long? = null + var timestamp: Long? = null, + @JsonField(name = ["reaction"]) + var reaction: String? = null ) : Parcelable { // This constructor is added to work with the 'com.bluelinelabs.logansquare.annotation.JsonObject' - constructor() : this(null, null, null, null, null, null, null) + constructor() : this(null, null, null, null, null, null, null, null) } diff --git a/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java b/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java index f1cf54e9b..07679304b 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java +++ b/app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java @@ -93,6 +93,12 @@ class CallParticipantMessageNotifier { } } + public void notifyReaction(String sessionId, String reaction) { + for (SignalingMessageReceiver.CallParticipantMessageListener listener : getListenersFor(sessionId)) { + listener.onReaction(reaction); + } + } + public synchronized void notifyUnshareScreen(String sessionId) { for (SignalingMessageReceiver.CallParticipantMessageListener listener : getListenersFor(sessionId)) { listener.onUnshareScreen(); diff --git a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java index ad18751a8..8853af425 100644 --- a/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java +++ b/app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java @@ -149,6 +149,7 @@ public abstract class SignalingMessageReceiver { */ public interface CallParticipantMessageListener { void onRaiseHand(boolean state, long timestamp); + void onReaction(String reaction); void onUnshareScreen(); } @@ -562,6 +563,57 @@ public abstract class SignalingMessageReceiver { return; } + if ("reaction".equals(type)) { + // Message schema (external signaling server): + // { + // "type": "message", + // "message": { + // "sender": { + // ... + // }, + // "data": { + // "to": #STRING#, + // "roomType": "video", + // "type": "reaction", + // "payload": { + // "reaction": #STRING#, + // }, + // "from": #STRING#, + // }, + // }, + // } + // + // Message schema (internal signaling server): + // { + // "type": "message", + // "data": { + // "to": #STRING#, + // "roomType": "video", + // "type": "reaction", + // "payload": { + // "reaction": #STRING#, + // }, + // "from": #STRING#, + // }, + // } + + NCMessagePayload payload = signalingMessage.getPayload(); + if (payload == null) { + // Broken message, this should not happen. + return; + } + + String reaction = payload.getReaction(); + if (reaction == null) { + // Broken message, this should not happen. + return; + } + + callParticipantMessageNotifier.notifyReaction(sessionId, reaction); + + return; + } + // "unshareScreen" messages are directly sent to the screen peer connection when the internal signaling // server is used, and to the room when the external signaling server is used. However, the (relevant) data // of the received message ("from" and "type") is the same in both cases. diff --git a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java index 96d90b23d..525e26d69 100644 --- a/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java +++ b/app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java @@ -84,6 +84,26 @@ public class SignalingMessageReceiverCallParticipantTest { verify(mockedCallParticipantMessageListener, only()).onRaiseHand(true, 4815162342L); } + @Test + public void testCallParticipantMessageReaction() { + SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = + mock(SignalingMessageReceiver.CallParticipantMessageListener.class); + + signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId"); + + NCSignalingMessage signalingMessage = new NCSignalingMessage(); + signalingMessage.setFrom("theSessionId"); + signalingMessage.setType("reaction"); + signalingMessage.setRoomType("theRoomType"); + NCMessagePayload messagePayload = new NCMessagePayload(); + messagePayload.setType("reaction"); + messagePayload.setReaction("theReaction"); + signalingMessage.setPayload(messagePayload); + signalingMessageReceiver.processSignalingMessage(signalingMessage); + + verify(mockedCallParticipantMessageListener, only()).onReaction("theReaction"); + } + @Test public void testCallParticipantMessageUnshareScreen() { SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener = From 92d655080d2317568f846d6dcd8f64eced7041e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 4 May 2023 05:42:51 +0200 Subject: [PATCH 2/3] Add "reaction" event to CallParticipantModel observer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CallParticipantModel observer now also emits one-time events that are not reflected in the model state. Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/activities/CallActivity.java | 4 ++++ .../talk/adapters/ParticipantDisplayItem.java | 11 ++++++++++- .../com/nextcloud/talk/call/CallParticipant.java | 1 + .../nextcloud/talk/call/CallParticipantModel.java | 6 +++++- .../talk/call/CallParticipantModelNotifier.java | 12 ++++++++++++ .../talk/call/MutableCallParticipantModel.java | 4 ++++ .../nextcloud/talk/call/CallParticipantModelTest.kt | 7 +++++++ 7 files changed, 43 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 2791e23f4..ed4f091f8 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2861,6 +2861,10 @@ public class CallActivity extends CallBaseActivity { addParticipantDisplayItem(callParticipantModel, "screen"); } } + + @Override + public void onReaction(String reaction) { + } } private class InternalSignalingMessageSender implements SignalingMessageSender { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index e488b9384..758f86a7b 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -34,7 +34,16 @@ public class ParticipantDisplayItem { private final CallParticipantModel callParticipantModel; - private final CallParticipantModel.Observer callParticipantModelObserver = this::updateFromModel; + private final CallParticipantModel.Observer callParticipantModelObserver = new CallParticipantModel.Observer() { + @Override + public void onChange() { + updateFromModel(); + } + + @Override + public void onReaction(String reaction) { + } + }; private String userId; private PeerConnection.IceConnectionState iceConnectionState; diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java index e0f4df295..02d3c8d2b 100644 --- a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java @@ -42,6 +42,7 @@ public class CallParticipant { @Override public void onReaction(String reaction) { + callParticipantModel.emitReaction(reaction); } @Override diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java index c8b1491d9..4f29420d1 100644 --- a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java @@ -42,11 +42,15 @@ import java.util.Objects; * Getters called after receiving a notification are guaranteed to provide at least the value that triggered the * notification, but it may return even a more up to date one (so getting the value again on the following * notification may return the same value as before). + * + * Besides onChange(), which notifies about changes in the model values, CallParticipantModel.Observer provides + * additional methods to be notified about one-time events that are not reflected in the model values, like reactions. */ public class CallParticipantModel { public interface Observer { void onChange(); + void onReaction(String reaction); } protected class Data { @@ -68,7 +72,7 @@ public class CallParticipantModel { } } - private final CallParticipantModelNotifier callParticipantModelNotifier = new CallParticipantModelNotifier(); + protected final CallParticipantModelNotifier callParticipantModelNotifier = new CallParticipantModelNotifier(); protected final String sessionId; diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java index ddf30c1d7..e0b6904fc 100644 --- a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java @@ -83,4 +83,16 @@ class CallParticipantModelNotifier { } } } + + public synchronized void notifyReaction(String reaction) { + for (CallParticipantModelObserverOn observerOn : new ArrayList<>(callParticipantModelObserversOn)) { + if (observerOn.handler == null || observerOn.handler.getLooper() == Looper.myLooper()) { + observerOn.observer.onReaction(reaction); + } else { + observerOn.handler.post(() -> { + observerOn.observer.onReaction(reaction); + }); + } + } + } } diff --git a/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java b/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java index 2772b0030..f57bd16d7 100644 --- a/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java +++ b/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java @@ -72,4 +72,8 @@ public class MutableCallParticipantModel extends CallParticipantModel { public void setScreenMediaStream(MediaStream screenMediaStream) { this.screenMediaStream.setValue(screenMediaStream); } + + public void emitReaction(String reaction) { + this.callParticipantModelNotifier.notifyReaction(reaction); + } } diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantModelTest.kt b/app/src/test/java/com/nextcloud/talk/call/CallParticipantModelTest.kt index efba5bd4e..29e144bb9 100644 --- a/app/src/test/java/com/nextcloud/talk/call/CallParticipantModelTest.kt +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantModelTest.kt @@ -55,4 +55,11 @@ class CallParticipantModelTest { callParticipantModel!!.setRaisedHand(true, 4815162342L) Mockito.verify(mockedCallParticipantModelObserver, Mockito.only())?.onChange() } + + @Test + fun testEmitReaction() { + callParticipantModel!!.addObserver(mockedCallParticipantModelObserver) + callParticipantModel!!.emitReaction("theReaction") + Mockito.verify(mockedCallParticipantModelObserver, Mockito.only())?.onReaction("theReaction") + } } From efab4cb66479ac5a759eda7a17e1d009a745188a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 4 May 2023 05:45:23 +0200 Subject: [PATCH 3/3] Handle raised hands from the call participants rather than the signaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although listening from the signaling was working fine and this unfortunately adds a lot of extra code it is conceptually "more correct", as the UI should not directly deal with the signaling if there is a higher abstraction available. Nevertheless, this should ease adding other similar changes, like reactions. Note that although there were already listeners for CallParticipantModel.Observer in the CallActivity they were not reused, as they handle totally unrelated things. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 57 +++++++++++++++---- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ed4f091f8..b7a7b59d9 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -296,6 +296,10 @@ public class CallActivity extends CallBaseActivity { private Handler screenParticipantDisplayItemManagersHandler = new Handler(Looper.getMainLooper()); + private Map callParticipantEventDisplayers = new HashMap<>(); + + private Handler callParticipantEventDisplayersHandler = new Handler(Looper.getMainLooper()); + private CallParticipantList.Observer callParticipantListObserver = new CallParticipantList.Observer() { @Override public void onCallParticipantsChanged(Collection joined, Collection updated, @@ -2248,6 +2252,11 @@ public class CallActivity extends CallBaseActivity { screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); + CallParticipantEventDisplayer callParticipantEventDisplayer = + new CallParticipantEventDisplayer(callParticipantModel); + callParticipantEventDisplayers.put(sessionId, callParticipantEventDisplayer); + callParticipantModel.addObserver(callParticipantEventDisplayer, callParticipantEventDisplayersHandler); + runOnUiThread(() -> { addParticipantDisplayItem(callParticipantModel, "video"); }); @@ -2288,6 +2297,10 @@ public class CallActivity extends CallBaseActivity { screenParticipantDisplayItemManagers.remove(sessionId); callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); + CallParticipantEventDisplayer callParticipantEventDisplayer = + callParticipantEventDisplayers.remove(sessionId); + callParticipant.getCallParticipantModel().removeObserver(callParticipantEventDisplayer); + callParticipant.destroy(); SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); @@ -2793,16 +2806,6 @@ public class CallActivity extends CallBaseActivity { @Override public void onRaiseHand(boolean state, long timestamp) { - if (state) { - CallParticipant participant = callParticipants.get(sessionId); - if (participant != null) { - String nick = participant.getCallParticipantModel().getNick(); - runOnUiThread(() -> Toast.makeText( - context, - String.format(context.getResources().getString(R.string.nc_call_raised_hand), nick), - Toast.LENGTH_LONG).show()); - } - } } @Override @@ -2867,6 +2870,40 @@ public class CallActivity extends CallBaseActivity { } } + private class CallParticipantEventDisplayer implements CallParticipantModel.Observer { + + private final CallParticipantModel callParticipantModel; + + private boolean raisedHand; + + private CallParticipantEventDisplayer(CallParticipantModel callParticipantModel) { + this.callParticipantModel = callParticipantModel; + this.raisedHand = callParticipantModel.getRaisedHand() != null ? + callParticipantModel.getRaisedHand().getState() : false; + } + + @Override + public void onChange() { + if (callParticipantModel.getRaisedHand() == null || !callParticipantModel.getRaisedHand().getState()) { + raisedHand = false; + + return; + } + + if (raisedHand) { + return; + } + raisedHand = true; + + String nick = callParticipantModel.getNick(); + Toast.makeText(context, String.format(context.getResources().getString(R.string.nc_call_raised_hand), nick), Toast.LENGTH_LONG).show(); + } + + @Override + public void onReaction(String reaction) { + } + } + private class InternalSignalingMessageSender implements SignalingMessageSender { @Override