From 346089cecf80a97c9273ef4fbef60c095b288436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Kr=C3=BCger?= Date: Tue, 29 Mar 2022 16:29:50 +0200 Subject: [PATCH 1/2] Use Transceivers instead of constraints to discard received video tracks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the migration from WebRTC from plan b to unified plan in commit 86f20dc the media constraints are ignored for creating an answer on a peer connection: > Offer to Receive Options/Constraints > > These constraints are passed to PeerConnection’s methods for creating SDP to > add placeholder media sections in case a sending track would not have already > created one. > > With Unified Plan semantics, these are still supported for creating offers > using a backwards compatibility shim (basically creating an RtpTransceiver if > one doesn’t exist). It is recommended to switch to the RtpTransceiver API > (example below). _They are no longer supported when creating answers._ > > [1] Because of that now all transceivers of type VIDEO will be stopped if it is only an audio call. After stopping the transceivers the status for the video tracks will be automatically set to ENDED. In the 'ParticipantsAdapter' it will be checked if the video tracks are ENDED. In that case the user avatar will be shown like before. Resolves: #1852 See: #1773, commit 86f20dc, [1] [1] https://docs.google.com/document/d/1PPHWV6108znP1tk_rkCnyagH9FK205hHeE9k5mhUzOg/edit#heading=h.9dcmkavg608r Signed-off-by: Tim Krüger --- .../talk/adapters/ParticipantsAdapter.java | 17 ++++++++++- .../talk/webrtc/PeerConnectionWrapper.java | 28 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java index 9a383552a..2b835c3e2 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java @@ -19,6 +19,7 @@ import com.nextcloud.talk.activities.CallActivity; import com.nextcloud.talk.utils.DisplayUtils; import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; import org.webrtc.RendererCommon; import org.webrtc.SurfaceViewRenderer; import org.webrtc.VideoTrack; @@ -140,7 +141,21 @@ public class ParticipantsAdapter extends BaseAdapter { } private boolean hasVideoStream(ParticipantDisplayItem participantDisplayItem, MediaStream mediaStream) { - return mediaStream != null && mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0 && participantDisplayItem.isStreamEnabled(); + if (!participantDisplayItem.isStreamEnabled()) { + return false; + } + + if (mediaStream == null || mediaStream.videoTracks == null) { + return false; + } + + for (VideoTrack t : mediaStream.videoTracks) { + if (MediaStreamTrack.State.LIVE == t.state()) { + return true; + } + } + + return false; } private int scaleGridViewItemHeight() { diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 1fd33cc1b..7dc57fd17 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -44,9 +44,11 @@ import org.webrtc.DataChannel; import org.webrtc.IceCandidate; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.RtpReceiver; +import org.webrtc.RtpTransceiver; import org.webrtc.SdpObserver; import org.webrtc.SessionDescription; import org.webrtc.VideoTrack; @@ -256,6 +258,15 @@ public class PeerConnectionWrapper { return isMCUPublisher; } + private boolean isAudioCallOnly() { + for (MediaConstraints.KeyValuePair keyValuePair : mediaConstraints.mandatory) { + if ("OfferToReceiveVideo".equals(keyValuePair.getKey())) { + return !Boolean.parseBoolean(keyValuePair.getValue()); + } + } + return false; + } + private class MagicDataChannelObserver implements DataChannel.Observer { @Override @@ -444,7 +455,22 @@ public class PeerConnectionWrapper { public void onSetSuccess() { if (peerConnection != null) { if (peerConnection.getLocalDescription() == null) { - peerConnection.createAnswer(magicSdpObserver, mediaConstraints); + + if (isAudioCallOnly()) { + for (RtpTransceiver t : peerConnection.getTransceivers()) { + if (t.getMediaType().equals(MediaStreamTrack.MediaType.MEDIA_TYPE_VIDEO)) { + t.stop(); + } + } + Log.d(TAG, "Stop all Transceivers for MEDIA_TYPE_VIDEO."); + } + + /* + Passed 'MediaConstraints' will be ignored by WebRTC when using UNIFIED PLAN. + See for details: https://docs.google.com/document/d/1PPHWV6108znP1tk_rkCnyagH9FK205hHeE9k5mhUzOg/edit#heading=h.9dcmkavg608r + */ + peerConnection.createAnswer(magicSdpObserver, new MediaConstraints()); + } if (peerConnection.getRemoteDescription() != null) { From 49fd2640bf858bd739a006b317550814cf17b233 Mon Sep 17 00:00:00 2001 From: Marcel Hibbe Date: Wed, 30 Mar 2022 12:08:40 +0200 Subject: [PATCH 2/2] rename method Signed-off-by: Marcel Hibbe --- .../java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 7dc57fd17..486e39713 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -258,7 +258,7 @@ public class PeerConnectionWrapper { return isMCUPublisher; } - private boolean isAudioCallOnly() { + private boolean shouldReceiveVideo() { for (MediaConstraints.KeyValuePair keyValuePair : mediaConstraints.mandatory) { if ("OfferToReceiveVideo".equals(keyValuePair.getKey())) { return !Boolean.parseBoolean(keyValuePair.getValue()); @@ -456,7 +456,7 @@ public class PeerConnectionWrapper { if (peerConnection != null) { if (peerConnection.getLocalDescription() == null) { - if (isAudioCallOnly()) { + if (shouldReceiveVideo()) { for (RtpTransceiver t : peerConnection.getTransceivers()) { if (t.getMediaType().equals(MediaStreamTrack.MediaType.MEDIA_TYPE_VIDEO)) { t.stop();