From 9f445efc1cff71d2ca0b4ab9815881dc14db3f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 16 Sep 2022 00:03:39 +0200 Subject: [PATCH 1/6] Post event when a participant is connected and disconnected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RTCPeerConnections have several states but, for simplicity, for now the events posted reflect only if the connection is fully established or not (which includes both a broken connection or an established connection that is unstable or being updated), which is enough for a basic information about the connection state in the UI. Signed-off-by: Daniel Calviño Sánchez --- .../nextcloud/talk/events/PeerConnectionEvent.java | 2 +- .../talk/webrtc/PeerConnectionWrapper.java | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java index 4e48929f7..c6722732e 100644 --- a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java @@ -120,6 +120,6 @@ public class PeerConnectionEvent { } public enum PeerConnectionEventType { - PEER_CONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE, PUBLISHER_FAILED + PEER_CONNECTED, PEER_DISCONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE, PUBLISHER_FAILED } } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 07a8f4df2..64e7c0ddf 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -344,6 +344,8 @@ public class PeerConnectionWrapper { Log.d("iceConnectionChangeTo: ", iceConnectionState.name() + " over " + peerConnection.hashCode() + " " + sessionId); if (iceConnectionState.equals(PeerConnection.IceConnectionState.CONNECTED)) { + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED, + sessionId, null, null, null)); if (!isMCUPublisher) { EventBus.getDefault().post(new MediaStreamEvent(remoteStream, sessionId, videoStreamType)); @@ -352,11 +354,20 @@ public class PeerConnectionWrapper { if (hasInitiated) { sendInitialMediaStatus(); } - + } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.COMPLETED)) { + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED, + sessionId, null, null, null)); } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.CLOSED)) { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType .PEER_CLOSED, sessionId, null, null, videoStreamType)); + } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.DISCONNECTED) || + iceConnectionState.equals(PeerConnection.IceConnectionState.NEW) || + iceConnectionState.equals(PeerConnection.IceConnectionState.CHECKING)) { + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED, + sessionId, null, null, null)); } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.FAILED)) { + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED, + sessionId, null, null, null)); if (isMCUPublisher) { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED, sessionId, null, null, null)); } From 719797b1d12bc89a44edf64acb18427127578c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 16 Sep 2022 00:05:02 +0200 Subject: [PATCH 2/6] Notify a data set change only if the properties are set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that this still notifies a data set change if the properties are set to the same value that they had already, but at least it is not notified when no data was actually changed. Signed-off-by: Daniel Calviño Sánchez --- .../java/com/nextcloud/talk/activities/CallActivity.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 0f691f353..be0d6ac1b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2070,23 +2070,20 @@ public class CallActivity extends CallBaseActivity { PeerConnectionEvent.PeerConnectionEventType.NICK_CHANGE) { if (participantDisplayItems.get(sessionId) != null) { participantDisplayItems.get(sessionId).setNick(peerConnectionEvent.getNick()); + participantsAdapter.notifyDataSetChanged(); } - participantsAdapter.notifyDataSetChanged(); - } else if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.VIDEO_CHANGE && !isVoiceOnlyCall) { if (participantDisplayItems.get(sessionId) != null) { participantDisplayItems.get(sessionId).setStreamEnabled(peerConnectionEvent.getChangeValue()); + participantsAdapter.notifyDataSetChanged(); } - participantsAdapter.notifyDataSetChanged(); - } else if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.AUDIO_CHANGE) { if (participantDisplayItems.get(sessionId) != null) { participantDisplayItems.get(sessionId).setAudioEnabled(peerConnectionEvent.getChangeValue()); + participantsAdapter.notifyDataSetChanged(); } - participantsAdapter.notifyDataSetChanged(); - } else if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED) { currentCallStatus = CallStatus.PUBLISHER_FAILED; From c2ef651ce3110259991fc711aa7540464159188e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 16 Sep 2022 00:09:41 +0200 Subject: [PATCH 3/6] Store whether a participant is connected or not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 25 +++++++++++++++++-- .../talk/adapters/ParticipantDisplayItem.java | 12 ++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index be0d6ac1b..0c0a2c469 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2050,6 +2050,18 @@ public class CallActivity extends CallBaseActivity { String sessionId = peerConnectionEvent.getSessionId(); if (peerConnectionEvent.getPeerConnectionEventType() == + PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED) { + if (participantDisplayItems.get(sessionId) != null) { + participantDisplayItems.get(sessionId).setConnected(true); + participantsAdapter.notifyDataSetChanged(); + } + } else if (peerConnectionEvent.getPeerConnectionEventType() == + PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED) { + if (participantDisplayItems.get(sessionId) != null) { + participantDisplayItems.get(sessionId).setConnected(false); + participantsAdapter.notifyDataSetChanged(); + } + } else if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.PEER_CLOSED) { endPeerConnection(sessionId, VIDEO_STREAM_TYPE_SCREEN.equals(peerConnectionEvent.getVideoStreamType())); } else if (peerConnectionEvent.getPeerConnectionEventType() == @@ -2239,12 +2251,20 @@ public class CallActivity extends CallBaseActivity { String session, boolean videoStreamEnabled, String videoStreamType) { + PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session, + videoStreamType); + + boolean connected = false; + if (peerConnectionWrapper != null) { + PeerConnection.IceConnectionState iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); + connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + } + String nick; if (hasExternalSignalingServer) { nick = webSocketClient.getDisplayNameForSession(session); } else { - PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session, - videoStreamType); nick = peerConnectionWrapper != null ? peerConnectionWrapper.getNick() : ""; } @@ -2268,6 +2288,7 @@ public class CallActivity extends CallBaseActivity { ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(userId, session, + connected, nick, urlForAvatar, mediaStream, diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index aefb742d8..adc75a338 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -6,6 +6,7 @@ import org.webrtc.MediaStream; public class ParticipantDisplayItem { private String userId; private String session; + private boolean connected; private String nick; private String urlForAvatar; private MediaStream mediaStream; @@ -14,9 +15,10 @@ public class ParticipantDisplayItem { private EglBase rootEglBase; private boolean isAudioEnabled; - public ParticipantDisplayItem(String userId, String session, String nick, String urlForAvatar, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { + public ParticipantDisplayItem(String userId, String session, boolean connected, String nick, String urlForAvatar, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { this.userId = userId; this.session = session; + this.connected = connected; this.nick = nick; this.urlForAvatar = urlForAvatar; this.mediaStream = mediaStream; @@ -41,6 +43,14 @@ public class ParticipantDisplayItem { this.session = session; } + public boolean isConnected() { + return connected; + } + + public void setConnected(boolean connected) { + this.connected = connected; + } + public String getNick() { return nick; } From a76e519219b9211d45c122bf65bcff27d1495d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 16 Sep 2022 00:14:10 +0200 Subject: [PATCH 4/6] Show progress bar on remote participant when not connected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/adapters/ParticipantsAdapter.java | 8 ++++++++ app/src/main/res/layout/call_item.xml | 7 +++++++ app/src/main/res/values/dimens.xml | 1 + 3 files changed, 16 insertions(+) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java index 20b708ff1..bd09717d6 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java @@ -8,6 +8,7 @@ import android.view.ViewGroup; import android.widget.BaseAdapter; import android.widget.ImageView; import android.widget.LinearLayout; +import android.widget.ProgressBar; import android.widget.RelativeLayout; import android.widget.TextView; @@ -96,6 +97,13 @@ public class ParticipantsAdapter extends BaseAdapter { surfaceViewRenderer = convertView.findViewById(R.id.surface_view); } + ProgressBar progressBar = convertView.findViewById(R.id.participant_progress_bar); + if (!participantDisplayItem.isConnected()) { + progressBar.setVisibility(View.VISIBLE); + } else { + progressBar.setVisibility(View.GONE); + } + ViewGroup.LayoutParams layoutParams = convertView.getLayoutParams(); layoutParams.height = scaleGridViewItemHeight(); convertView.setLayoutParams(layoutParams); diff --git a/app/src/main/res/layout/call_item.xml b/app/src/main/res/layout/call_item.xml index 362b071a8..41f175472 100644 --- a/app/src/main/res/layout/call_item.xml +++ b/app/src/main/res/layout/call_item.xml @@ -71,4 +71,11 @@ android:visibility="invisible" tools:visibility="visible" /> + + diff --git a/app/src/main/res/values/dimens.xml b/app/src/main/res/values/dimens.xml index c5d27191c..69f5faaaf 100644 --- a/app/src/main/res/values/dimens.xml +++ b/app/src/main/res/values/dimens.xml @@ -66,6 +66,7 @@ 180dp 110dp + 48dp 0dp 52dp From 2f44e6fd19dc787d9aacb5eeea432d37f3fa6d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 16 Sep 2022 00:15:56 +0200 Subject: [PATCH 5/6] Show progress bar on local participant when not connected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 19 +++++++++++++++++-- app/src/main/res/layout/call_activity.xml | 8 ++++++++ app/src/main/res/values/dimens.xml | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 0c0a2c469..ca15b32cc 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2008,6 +2008,17 @@ public class CallActivity extends CallBaseActivity { updateSelfVideoViewPosition(); } + private void updateSelfVideoViewConnected(boolean connected) { + // FIXME In voice only calls there is no video view, so the progress bar would appear floating in the middle of + // nowhere. However, a way to signal that the local participant is not connected to the HPB is still need in + // that case. + if (!connected && !isVoiceOnlyCall) { + binding.selfVideoViewProgressBar.setVisibility(View.VISIBLE); + } else { + binding.selfVideoViewProgressBar.setVisibility(View.GONE); + } + } + private void updateSelfVideoViewPosition() { Log.d(TAG, "updateSelfVideoViewPosition"); if (!isInPipMode) { @@ -2051,13 +2062,17 @@ public class CallActivity extends CallBaseActivity { if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED) { - if (participantDisplayItems.get(sessionId) != null) { + if (webSocketClient != null && webSocketClient.getSessionId() == sessionId) { + updateSelfVideoViewConnected(true); + } else if (participantDisplayItems.get(sessionId) != null) { participantDisplayItems.get(sessionId).setConnected(true); participantsAdapter.notifyDataSetChanged(); } } else if (peerConnectionEvent.getPeerConnectionEventType() == PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED) { - if (participantDisplayItems.get(sessionId) != null) { + if (webSocketClient != null && webSocketClient.getSessionId() == sessionId) { + updateSelfVideoViewConnected(false); + } else if (participantDisplayItems.get(sessionId) != null) { participantDisplayItems.get(sessionId).setConnected(false); participantsAdapter.notifyDataSetChanged(); } diff --git a/app/src/main/res/layout/call_activity.xml b/app/src/main/res/layout/call_activity.xml index 897c6ed06..ca5be5a8e 100644 --- a/app/src/main/res/layout/call_activity.xml +++ b/app/src/main/res/layout/call_activity.xml @@ -77,6 +77,14 @@ android:layout_marginBottom="20dp" app:placeholderImage="@drawable/ic_switch_video_white_24px" app:roundAsCircle="true" /> + + 180dp 110dp 48dp + 48dp 0dp 52dp From c3f1f6c3a8ff74930b91b5f00c6435186f5ec1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 16 Sep 2022 00:20:01 +0200 Subject: [PATCH 6/6] Show participant as soon as found rather than once connected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As the participants that are not connected yet are clearly marked as such now the participants are shown as soon as they are found rather than waiting for a connection to be established. There is a drawback, however; if a participant will never have a connection (for example, if the HPB is used and that participant does not have publishing permissions) the participant will be endlessly shown with the progress bar. Nevertheless, before they were not shown at all, which was probably even more puzzling. Signed-off-by: Daniel Calviño Sánchez --- .../java/com/nextcloud/talk/activities/CallActivity.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ca15b32cc..f88e1fb1f 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1832,6 +1832,14 @@ public class CallActivity extends CallBaseActivity { for (String sessionId : newSessions) { Log.d(TAG, " newSession joined: " + sessionId); getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); + + runOnUiThread(() -> { + setupVideoStreamForLayout( + null, + sessionId, + false, + VIDEO_STREAM_TYPE_VIDEO); + }); } if (newSessions.size() > 0 && !currentCallStatus.equals(CallStatus.IN_CONVERSATION)) {