mirror of
https://github.com/nextcloud/talk-android
synced 2025-06-21 12:39:58 +01:00
bad5b5f9bc
The updated libwebrtc will be load from the talk-clients-webrtc repository [1] and not from a dependency repository. The 'lint' task must now depend on the 'perBuild' task to make sure that the WebRTC library is downloaded from the configured URL. To get the new version to work a change is needed because of this announcement: > Google is planning to transition Chrome’s WebRTC implementation from the > current SDP format (called “Plan B”) to a standards conformant format > (“Unified Plan”, draft-ietf-rtcweb-jsep) over the next couple of quarters. > > - [2] Currently we are in phase 4 of the transition [3]. So actually 'Plan B' must be used to avoid bigger changes to our code base at this time. But 'Plan B' is depracted and will be removed in phase 5 of the transition [4]. Resolves: #1773 See: - [1] https://github.com/nextcloud-releases/talk-clients-webrtc - [2] https://webrtc.org/getting-started/unified-plan-transition-guide - [3] https://webrtc.org/getting-started/unified-plan-transition-guide#phase_4_make_%E2%80%9Cplan_b%E2%80%9D_throw - [4] https://webrtc.org/getting-started/unified-plan-transition-guide#phase_5_remove_%E2%80%9Cplan_b%E2%80%9D Signed-off-by: Tim Krüger <t@timkrueger.me>
485 lines
19 KiB
Java
485 lines
19 KiB
Java
/*
|
|
* Nextcloud Talk application
|
|
*
|
|
* @author Mario Danic
|
|
* Copyright (C) 2017 Mario Danic <mario@lovelyhq.com>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*
|
|
*/
|
|
|
|
package com.nextcloud.talk.webrtc;
|
|
|
|
import android.content.Context;
|
|
import android.text.TextUtils;
|
|
import android.util.Log;
|
|
|
|
import com.bluelinelabs.logansquare.LoganSquare;
|
|
import com.nextcloud.talk.R;
|
|
import com.nextcloud.talk.application.NextcloudTalkApplication;
|
|
import com.nextcloud.talk.events.MediaStreamEvent;
|
|
import com.nextcloud.talk.events.PeerConnectionEvent;
|
|
import com.nextcloud.talk.events.SessionDescriptionSendEvent;
|
|
import com.nextcloud.talk.events.WebSocketCommunicationEvent;
|
|
import com.nextcloud.talk.models.json.signaling.DataChannelMessage;
|
|
import com.nextcloud.talk.models.json.signaling.DataChannelMessageNick;
|
|
import com.nextcloud.talk.models.json.signaling.NCIceCandidate;
|
|
|
|
import org.greenrobot.eventbus.EventBus;
|
|
import org.webrtc.DataChannel;
|
|
import org.webrtc.IceCandidate;
|
|
import org.webrtc.MediaConstraints;
|
|
import org.webrtc.MediaStream;
|
|
import org.webrtc.PeerConnection;
|
|
import org.webrtc.PeerConnectionFactory;
|
|
import org.webrtc.RtpReceiver;
|
|
import org.webrtc.SdpObserver;
|
|
import org.webrtc.SessionDescription;
|
|
|
|
import java.io.IOException;
|
|
import java.nio.ByteBuffer;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
|
|
import javax.inject.Inject;
|
|
|
|
import androidx.annotation.Nullable;
|
|
import autodagger.AutoInjector;
|
|
|
|
@AutoInjector(NextcloudTalkApplication.class)
|
|
public class MagicPeerConnectionWrapper {
|
|
private static final String TAG = "MagicPeerConWrapper";
|
|
|
|
private List<IceCandidate> iceCandidates = new ArrayList<>();
|
|
private PeerConnection peerConnection;
|
|
private String sessionId;
|
|
private String nick;
|
|
private MediaConstraints sdpConstraints;
|
|
private DataChannel magicDataChannel;
|
|
private MagicSdpObserver magicSdpObserver;
|
|
private MediaStream remoteMediaStream;
|
|
|
|
private boolean remoteVideoOn;
|
|
private boolean remoteAudioOn;
|
|
|
|
private boolean hasInitiated;
|
|
|
|
private MediaStream localMediaStream;
|
|
private boolean isMCUPublisher;
|
|
private boolean hasMCU;
|
|
private String videoStreamType;
|
|
|
|
private int connectionAttempts = 0;
|
|
private PeerConnection.IceConnectionState peerIceConnectionState;
|
|
|
|
@Inject
|
|
Context context;
|
|
|
|
public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
|
|
List<PeerConnection.IceServer> iceServerList,
|
|
MediaConstraints sdpConstraints,
|
|
String sessionId, String localSession, @Nullable MediaStream mediaStream,
|
|
boolean isMCUPublisher, boolean hasMCU, String videoStreamType) {
|
|
|
|
NextcloudTalkApplication.Companion.getSharedApplication().getComponentApplication().inject(this);
|
|
|
|
this.localMediaStream = mediaStream;
|
|
this.videoStreamType = videoStreamType;
|
|
this.hasMCU = hasMCU;
|
|
|
|
this.sessionId = sessionId;
|
|
this.sdpConstraints = sdpConstraints;
|
|
|
|
magicSdpObserver = new MagicSdpObserver();
|
|
hasInitiated = sessionId.compareTo(localSession) < 0;
|
|
this.isMCUPublisher = isMCUPublisher;
|
|
|
|
PeerConnection.RTCConfiguration configuration = new PeerConnection.RTCConfiguration(iceServerList);
|
|
configuration.sdpSemantics = PeerConnection.SdpSemantics.PLAN_B;
|
|
|
|
peerConnection = peerConnectionFactory.createPeerConnection(configuration, sdpConstraints,
|
|
new MagicPeerConnectionObserver());
|
|
|
|
if (peerConnection != null) {
|
|
if (localMediaStream != null) {
|
|
peerConnection.addStream(localMediaStream);
|
|
}
|
|
|
|
if (hasMCU || hasInitiated) {
|
|
DataChannel.Init init = new DataChannel.Init();
|
|
init.negotiated = false;
|
|
magicDataChannel = peerConnection.createDataChannel("status", init);
|
|
magicDataChannel.registerObserver(new MagicDataChannelObserver());
|
|
if (isMCUPublisher) {
|
|
peerConnection.createOffer(magicSdpObserver, sdpConstraints);
|
|
} else if (hasMCU && this.videoStreamType.equals("video")) {
|
|
// If the connection type is "screen" the client sharing the screen will send an
|
|
// offer; offers should be requested only for videos.
|
|
HashMap<String, String> hashMap = new HashMap<>();
|
|
hashMap.put("sessionId", sessionId);
|
|
EventBus.getDefault().post(new WebSocketCommunicationEvent("peerReadyForRequestingOffer", hashMap));
|
|
} else if (!hasMCU && hasInitiated) {
|
|
peerConnection.createOffer(magicSdpObserver, sdpConstraints);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public String getVideoStreamType() {
|
|
return videoStreamType;
|
|
}
|
|
|
|
public void removePeerConnection() {
|
|
if (magicDataChannel != null) {
|
|
magicDataChannel.dispose();
|
|
magicDataChannel = null;
|
|
}
|
|
|
|
if (peerConnection != null) {
|
|
if (localMediaStream != null) {
|
|
peerConnection.removeStream(localMediaStream);
|
|
}
|
|
|
|
peerConnection.close();
|
|
peerConnection = null;
|
|
}
|
|
}
|
|
|
|
public void drainIceCandidates() {
|
|
|
|
if (peerConnection != null) {
|
|
for (IceCandidate iceCandidate : iceCandidates) {
|
|
peerConnection.addIceCandidate(iceCandidate);
|
|
}
|
|
|
|
iceCandidates = new ArrayList<>();
|
|
}
|
|
}
|
|
|
|
public MagicSdpObserver getMagicSdpObserver() {
|
|
return magicSdpObserver;
|
|
}
|
|
|
|
public void addCandidate(IceCandidate iceCandidate) {
|
|
if (peerConnection != null && peerConnection.getRemoteDescription() != null) {
|
|
peerConnection.addIceCandidate(iceCandidate);
|
|
} else {
|
|
iceCandidates.add(iceCandidate);
|
|
}
|
|
}
|
|
|
|
public void sendNickChannelData(DataChannelMessageNick dataChannelMessage) {
|
|
ByteBuffer buffer;
|
|
if (magicDataChannel != null) {
|
|
try {
|
|
buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes());
|
|
magicDataChannel.send(new DataChannel.Buffer(buffer, false));
|
|
} catch (IOException e) {
|
|
Log.d(TAG, "Failed to send channel data, attempting regular " + dataChannelMessage);
|
|
}
|
|
}
|
|
}
|
|
|
|
public void sendChannelData(DataChannelMessage dataChannelMessage) {
|
|
ByteBuffer buffer;
|
|
if (magicDataChannel != null) {
|
|
try {
|
|
buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes());
|
|
magicDataChannel.send(new DataChannel.Buffer(buffer, false));
|
|
} catch (IOException e) {
|
|
Log.d(TAG, "Failed to send channel data, attempting regular " + dataChannelMessage);
|
|
}
|
|
}
|
|
}
|
|
|
|
public PeerConnection getPeerConnection() {
|
|
return peerConnection;
|
|
}
|
|
|
|
public String getSessionId() {
|
|
return sessionId;
|
|
}
|
|
|
|
public void setSessionId(String sessionId) {
|
|
this.sessionId = sessionId;
|
|
}
|
|
|
|
public String getNick() {
|
|
if (!TextUtils.isEmpty(nick)) {
|
|
return nick;
|
|
} else {
|
|
return NextcloudTalkApplication.Companion.getSharedApplication().getString(R.string.nc_nick_guest);
|
|
}
|
|
}
|
|
|
|
public void setNick(String nick) {
|
|
this.nick = nick;
|
|
}
|
|
|
|
private void sendInitialMediaStatus() {
|
|
if (localMediaStream != null) {
|
|
if (localMediaStream.videoTracks.size() == 1 && localMediaStream.videoTracks.get(0).enabled()) {
|
|
sendChannelData(new DataChannelMessage("videoOn"));
|
|
} else {
|
|
sendChannelData(new DataChannelMessage("videoOff"));
|
|
}
|
|
|
|
if (localMediaStream.audioTracks.size() == 1 && localMediaStream.audioTracks.get(0).enabled()) {
|
|
sendChannelData(new DataChannelMessage("audioOn"));
|
|
} else {
|
|
sendChannelData(new DataChannelMessage("audioOff"));
|
|
}
|
|
}
|
|
}
|
|
|
|
public boolean isMCUPublisher() {
|
|
return isMCUPublisher;
|
|
}
|
|
|
|
private class MagicDataChannelObserver implements DataChannel.Observer {
|
|
|
|
@Override
|
|
public void onBufferedAmountChange(long l) {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void onStateChange() {
|
|
if (magicDataChannel != null && magicDataChannel.state().equals(DataChannel.State.OPEN) &&
|
|
magicDataChannel.label().equals("status")) {
|
|
sendInitialMediaStatus();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void onMessage(DataChannel.Buffer buffer) {
|
|
if (buffer.binary) {
|
|
Log.d(TAG, "Received binary msg over " + TAG + " " + sessionId);
|
|
return;
|
|
}
|
|
|
|
ByteBuffer data = buffer.data;
|
|
final byte[] bytes = new byte[data.capacity()];
|
|
data.get(bytes);
|
|
String strData = new String(bytes);
|
|
Log.d(TAG, "Got msg: " + strData + " over " + TAG + " " + sessionId);
|
|
|
|
try {
|
|
DataChannelMessage dataChannelMessage = LoganSquare.parse(strData, DataChannelMessage.class);
|
|
|
|
String internalNick;
|
|
if ("nickChanged".equals(dataChannelMessage.getType())) {
|
|
if (dataChannelMessage.getPayload() instanceof String) {
|
|
internalNick = (String) dataChannelMessage.getPayload();
|
|
if (!internalNick.equals(nick)) {
|
|
setNick(internalNick);
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.NICK_CHANGE, sessionId, getNick(), null, videoStreamType));
|
|
}
|
|
} else {
|
|
if (dataChannelMessage.getPayload() != null) {
|
|
HashMap<String, String> payloadHashMap = (HashMap<String, String>) dataChannelMessage.getPayload();
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.NICK_CHANGE, sessionId, payloadHashMap.get("name"), null, videoStreamType));
|
|
}
|
|
}
|
|
|
|
} else if ("audioOn".equals(dataChannelMessage.getType())) {
|
|
remoteAudioOn = true;
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType));
|
|
} else if ("audioOff".equals(dataChannelMessage.getType())) {
|
|
remoteAudioOn = false;
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType));
|
|
} else if ("videoOn".equals(dataChannelMessage.getType())) {
|
|
remoteVideoOn = true;
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType));
|
|
} else if ("videoOff".equals(dataChannelMessage.getType())) {
|
|
remoteVideoOn = false;
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType));
|
|
}
|
|
} catch (IOException e) {
|
|
Log.d(TAG, "Failed to parse data channel message");
|
|
}
|
|
}
|
|
}
|
|
|
|
private void restartIce() {
|
|
if (connectionAttempts <= 5) {
|
|
if (!hasMCU || isMCUPublisher) {
|
|
MediaConstraints.KeyValuePair iceRestartConstraint =
|
|
new MediaConstraints.KeyValuePair("IceRestart", "true");
|
|
|
|
if (sdpConstraints.mandatory.contains(iceRestartConstraint)) {
|
|
sdpConstraints.mandatory.add(iceRestartConstraint);
|
|
}
|
|
|
|
peerConnection.createOffer(magicSdpObserver, sdpConstraints);
|
|
} else {
|
|
// we have an MCU and this is not the publisher
|
|
// Do something if we have an MCU
|
|
}
|
|
|
|
connectionAttempts++;
|
|
}
|
|
}
|
|
|
|
private class MagicPeerConnectionObserver implements PeerConnection.Observer {
|
|
|
|
@Override
|
|
public void onSignalingChange(PeerConnection.SignalingState signalingState) {
|
|
}
|
|
|
|
@Override
|
|
public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
|
|
peerIceConnectionState = iceConnectionState;
|
|
|
|
Log.d("iceConnectionChangeTo: ", iceConnectionState.name() + " over " + peerConnection.hashCode() + " " + sessionId);
|
|
if (iceConnectionState.equals(PeerConnection.IceConnectionState.CONNECTED)) {
|
|
connectionAttempts = 0;
|
|
/*EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.PEER_CONNECTED, sessionId, null, null));*/
|
|
|
|
if (!isMCUPublisher) {
|
|
EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId, videoStreamType));
|
|
}
|
|
|
|
if (hasInitiated) {
|
|
sendInitialMediaStatus();
|
|
}
|
|
|
|
} else if (iceConnectionState.equals(PeerConnection.IceConnectionState.CLOSED)) {
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
|
|
.PEER_CLOSED, sessionId, null, null, videoStreamType));
|
|
connectionAttempts = 0;
|
|
} else if (iceConnectionState.equals(PeerConnection.IceConnectionState.FAILED)) {
|
|
/*if (MerlinTheWizard.isConnectedToInternet() && connectionAttempts < 5) {
|
|
restartIce();
|
|
}*/
|
|
if (isMCUPublisher) {
|
|
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED, sessionId, null, null, null));
|
|
}
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void onIceConnectionReceivingChange(boolean b) {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void onIceCandidate(IceCandidate iceCandidate) {
|
|
NCIceCandidate ncIceCandidate = new NCIceCandidate();
|
|
ncIceCandidate.setSdpMid(iceCandidate.sdpMid);
|
|
ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex);
|
|
ncIceCandidate.setCandidate(iceCandidate.sdp);
|
|
EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
|
|
"candidate", ncIceCandidate, videoStreamType));
|
|
}
|
|
|
|
@Override
|
|
public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void onAddStream(MediaStream mediaStream) {
|
|
remoteMediaStream = mediaStream;
|
|
}
|
|
|
|
@Override
|
|
public void onRemoveStream(MediaStream mediaStream) {
|
|
if (!isMCUPublisher) {
|
|
EventBus.getDefault().post(new MediaStreamEvent(null, sessionId, videoStreamType));
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void onDataChannel(DataChannel dataChannel) {
|
|
if (dataChannel.label().equals("status") || dataChannel.label().equals("JanusDataChannel")) {
|
|
magicDataChannel = dataChannel;
|
|
magicDataChannel.registerObserver(new MagicDataChannelObserver());
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void onRenegotiationNeeded() {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
|
|
}
|
|
}
|
|
|
|
private class MagicSdpObserver implements SdpObserver {
|
|
private static final String TAG = "MagicSdpObserver";
|
|
|
|
@Override
|
|
public void onCreateFailure(String s) {
|
|
Log.d(TAG, "SDPObserver createFailure: " + s + " over " + peerConnection.hashCode() + " " + sessionId);
|
|
|
|
}
|
|
|
|
@Override
|
|
public void onSetFailure(String s) {
|
|
Log.d(TAG,"SDPObserver setFailure: " + s + " over " + peerConnection.hashCode() + " " + sessionId);
|
|
}
|
|
|
|
@Override
|
|
public void onCreateSuccess(SessionDescription sessionDescription) {
|
|
SessionDescription sessionDescriptionWithPreferredCodec;
|
|
String sessionDescriptionStringWithPreferredCodec = MagicWebRTCUtils.preferCodec
|
|
(sessionDescription.description,
|
|
"H264", false);
|
|
sessionDescriptionWithPreferredCodec = new SessionDescription(
|
|
sessionDescription.type,
|
|
sessionDescriptionStringWithPreferredCodec);
|
|
|
|
|
|
EventBus.getDefault().post(new SessionDescriptionSendEvent(sessionDescriptionWithPreferredCodec, sessionId,
|
|
sessionDescription.type.canonicalForm().toLowerCase(), null, videoStreamType));
|
|
|
|
if (peerConnection != null) {
|
|
peerConnection.setLocalDescription(magicSdpObserver, sessionDescriptionWithPreferredCodec);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void onSetSuccess() {
|
|
if (peerConnection != null) {
|
|
if (peerConnection.getLocalDescription() == null) {
|
|
peerConnection.createAnswer(magicSdpObserver, sdpConstraints);
|
|
}
|
|
|
|
if (peerConnection.getRemoteDescription() != null) {
|
|
drainIceCandidates();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public PeerConnection.IceConnectionState getPeerIceConnectionState() {
|
|
return peerIceConnectionState;
|
|
}
|
|
} |