diff --git a/app/src/main/java/com/nextcloud/talk/activities/MagicCallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/MagicCallActivity.java index 51d52a877..ee1122420 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/MagicCallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/MagicCallActivity.java @@ -77,7 +77,7 @@ public class MagicCallActivity extends BaseActivity { ButterKnife.bind(this); router = Conductor.attachRouter(this, container, savedInstanceState); - router.setPopsLastView(true); + router.setPopsLastView(false); if (!router.hasRootController()) { if (getIntent().getBooleanExtra(BundleKeys.KEY_FROM_NOTIFICATION_START_CALL, false)) { diff --git a/app/src/main/java/com/nextcloud/talk/controllers/CallController.java b/app/src/main/java/com/nextcloud/talk/controllers/CallController.java index cff39dbfb..010d7b840 100644 --- a/app/src/main/java/com/nextcloud/talk/controllers/CallController.java +++ b/app/src/main/java/com/nextcloud/talk/controllers/CallController.java @@ -40,6 +40,7 @@ import android.view.MotionEvent; import android.view.View; import android.view.ViewGroup; import android.widget.*; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.appcompat.app.AppCompatActivity; @@ -47,6 +48,7 @@ import autodagger.AutoInjector; import butterknife.BindView; import butterknife.OnClick; import butterknife.OnLongClick; + import com.bluelinelabs.logansquare.LoganSquare; import com.facebook.drawee.backends.pipeline.Fresco; import com.facebook.drawee.interfaces.DraweeController; @@ -80,24 +82,30 @@ import com.nextcloud.talk.utils.singletons.ApplicationWideCurrentRoomHolder; import com.nextcloud.talk.utils.singletons.MerlinTheWizard; import com.nextcloud.talk.webrtc.*; import com.wooplr.spotlight.SpotlightView; + import io.reactivex.Observable; import io.reactivex.Observer; import io.reactivex.android.schedulers.AndroidSchedulers; import io.reactivex.disposables.Disposable; +import io.reactivex.functions.BooleanSupplier; +import io.reactivex.functions.Consumer; import io.reactivex.schedulers.Schedulers; import me.zhanghai.android.effortlesspermissions.AfterPermissionDenied; import me.zhanghai.android.effortlesspermissions.EffortlessPermissions; import me.zhanghai.android.effortlesspermissions.OpenAppDetailsDialogFragment; import okhttp3.Cache; + import org.apache.commons.lang3.StringEscapeUtils; import org.greenrobot.eventbus.EventBus; import org.greenrobot.eventbus.Subscribe; import org.greenrobot.eventbus.ThreadMode; import org.parceler.Parcel; import org.webrtc.*; + import pub.devrel.easypermissions.AfterPermissionGranted; import javax.inject.Inject; + import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; @@ -224,7 +232,7 @@ public class CallController extends BaseController { @Parcel public enum CallStatus { - CALLING, CALLING_TIMEOUT, ESTABLISHED, IN_CONVERSATION, RECONNECTING, OFFLINE, LEAVING + CALLING, CALLING_TIMEOUT, ESTABLISHED, IN_CONVERSATION, RECONNECTING, OFFLINE, LEAVING, PUBLISHER_FAILED } public CallController(Bundle args) { @@ -1174,13 +1182,18 @@ public class CallController extends BaseController { } private void setupAndInitiateWebSocketsConnection() { - webSocketConnectionHelper = new WebSocketConnectionHelper(); - webSocketClient = WebSocketConnectionHelper.getExternalSignalingInstanceForServer( - externalSignalingServer.getExternalSignalingServer(), - conversationUser, externalSignalingServer.getExternalSignalingTicket(), - TextUtils.isEmpty(credentials)); + if (webSocketConnectionHelper == null) { + webSocketConnectionHelper = new WebSocketConnectionHelper(); + } - joinRoomAndCall(); + if (webSocketClient == null) { + webSocketClient = WebSocketConnectionHelper.getExternalSignalingInstanceForServer( + externalSignalingServer.getExternalSignalingServer(), + conversationUser, externalSignalingServer.getExternalSignalingTicket(), + TextUtils.isEmpty(credentials)); + + joinRoomAndCall(); + } } private void initiateCall() { @@ -1190,22 +1203,23 @@ public class CallController extends BaseController { handleFromNotification(); } } + @Subscribe(threadMode = ThreadMode.BACKGROUND) public void onMessageEvent(WebSocketCommunicationEvent webSocketCommunicationEvent) { switch (webSocketCommunicationEvent.getType()) { case "hello": - if (!currentCallStatus.equals(CallStatus.RECONNECTING)) { - if (!webSocketCommunicationEvent.getHashMap().containsKey("oldResumeId")) { - initiateCall(); + if (!webSocketCommunicationEvent.getHashMap().containsKey("oldResumeId")) { + if (currentCallStatus.equals(CallStatus.RECONNECTING)) { + hangup(false); } else { - // do nothing, let's just continue + initiateCall(); } + } else { + } break; case "roomJoined": - if (hasExternalSignalingServer) { - startSendingNick(); - } + startSendingNick(); if (webSocketCommunicationEvent.getHashMap().get("roomToken").equals(roomToken)) { performCall(); @@ -1250,7 +1264,7 @@ public class CallController extends BaseController { private void receivedSignalingMessage(Signaling signaling) throws IOException { String messageType = signaling.getType(); - if (!isConnectionEstablished()) { + if (!isConnectionEstablished() && !currentCallStatus.equals(CallStatus.CALLING)) { return; } @@ -1393,7 +1407,7 @@ public class CallController extends BaseController { if (isMultiSession) { if (shutDownView && getActivity() != null) { getActivity().finish(); - } else if (!shutDownView && currentCallStatus.equals(CallStatus.RECONNECTING)) { + } else if (!shutDownView && (currentCallStatus.equals(CallStatus.RECONNECTING) || currentCallStatus.equals(CallStatus.PUBLISHER_FAILED))) { initiateCall(); } } else { @@ -1485,7 +1499,7 @@ public class CallController extends BaseController { // Calculate sessions that join the call newSessions.removeAll(oldSesssions); - if (!isConnectionEstablished()) { + if (!isConnectionEstablished() && !currentCallStatus.equals(CallStatus.CALLING)) { return; } @@ -1563,6 +1577,7 @@ public class CallController extends BaseController { if (hasMCU && publisher) { magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true, type); + } else if (hasMCU) { magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, iceServers, sdpConstraints, sessionId, callSession, null, false, true, type); @@ -1577,6 +1592,11 @@ public class CallController extends BaseController { } magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper); + + if (publisher) { + startSendingNick(); + } + return magicPeerConnectionWrapper; } } @@ -1687,11 +1707,16 @@ public class CallController extends BaseController { .PeerConnectionEventType.AUDIO_CHANGE)) { gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(), peerConnectionEvent.getChangeValue()); + } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED)) { + if (MerlinTheWizard.isConnectedToInternet()) { + currentCallStatus = CallStatus.RECONNECTING; + hangup(false); + } } } private void startSendingNick() { - DataChannelMessage dataChannelMessage = new DataChannelMessage(); + DataChannelMessageNick dataChannelMessage = new DataChannelMessageNick(); dataChannelMessage.setType("nickChanged"); HashMap nickChangedPayload = new HashMap<>(); nickChangedPayload.put("userid", conversationUser.getUserId()); @@ -1702,9 +1727,14 @@ public class CallController extends BaseController { int finalI = i; Observable .interval(1, TimeUnit.SECONDS) - .takeWhile(observer -> isConnectionEstablished()) + .repeat() .observeOn(Schedulers.io()) - .doOnNext(n -> magicPeerConnectionWrapperList.get(finalI).sendChannelData(dataChannelMessage)); + .doOnNext(new Consumer() { + @Override + public void accept(Long aLong) { + magicPeerConnectionWrapperList.get(finalI).sendNickChannelData(dataChannelMessage); + } + }).subscribe(); break; } @@ -2136,7 +2166,7 @@ public class CallController extends BaseController { mediaPlayer.setOnPreparedListener(mp -> mediaPlayer.start()); - mediaPlayer.prepareAsync(); + //mediaPlayer.prepareAsync(); } catch (IOException e) { Log.e(TAG, "Failed to play sound"); @@ -2199,16 +2229,20 @@ public class CallController extends BaseController { handler.removeCallbacksAndMessages(null); } - setCallState(CallStatus.RECONNECTING); - hangupNetworkCalls(false); + if (!hasMCU) { + setCallState(CallStatus.RECONNECTING); + hangupNetworkCalls(false); + } } else if (networkEvent.getNetworkConnectionEvent().equals(NetworkEvent.NetworkConnectionEvent.NETWORK_DISCONNECTED)) { if (handler != null) { handler.removeCallbacksAndMessages(null); } - setCallState(CallStatus.OFFLINE); - hangup(false); + if (!hasMCU) { + setCallState(CallStatus.OFFLINE); + hangup(false); + } } } } diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java index f85954c4a..c6ef6e0e8 100644 --- a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java @@ -41,6 +41,6 @@ public class PeerConnectionEvent { } public enum PeerConnectionEventType { - PEER_CONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE + PEER_CONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE, PUBLISHER_FAILED } } diff --git a/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessage.java b/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessage.java index 97c4594f4..a1843ca45 100644 --- a/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessage.java +++ b/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessage.java @@ -22,6 +22,10 @@ package com.nextcloud.talk.models.json.signaling; import com.bluelinelabs.logansquare.annotation.JsonField; import com.bluelinelabs.logansquare.annotation.JsonObject; +import com.nextcloud.talk.models.json.converters.ObjectParcelConverter; + +import org.parceler.ParcelPropertyConverter; + import lombok.Data; @Data @@ -30,6 +34,7 @@ public class DataChannelMessage { @JsonField(name = "type") String type; + @ParcelPropertyConverter(ObjectParcelConverter.class) @JsonField(name = "payload") Object payload; diff --git a/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessageNick.java b/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessageNick.java new file mode 100644 index 000000000..dcb41894f --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/models/json/signaling/DataChannelMessageNick.java @@ -0,0 +1,49 @@ +/* + * Nextcloud Talk application + * + * @author Mario Danic + * Copyright (C) 2017 Mario Danic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.nextcloud.talk.models.json.signaling; + +import com.bluelinelabs.logansquare.annotation.JsonField; +import com.bluelinelabs.logansquare.annotation.JsonObject; +import com.nextcloud.talk.models.json.converters.ObjectParcelConverter; + +import org.parceler.ParcelPropertyConverter; + +import java.util.HashMap; + +import lombok.Data; + +@Data +@JsonObject +public class DataChannelMessageNick { + @JsonField(name = "type") + String type; + + @ParcelPropertyConverter(ObjectParcelConverter.class) + @JsonField(name = "payload") + HashMap payload; + + public DataChannelMessageNick(String type) { + this.type = type; + } + + public DataChannelMessageNick() { + } +} diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java index 13016bc0b..bdc875ac1 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java @@ -26,6 +26,8 @@ import android.text.TextUtils; import android.util.Log; import androidx.annotation.Nullable; import autodagger.AutoInjector; + +import com.bluelinelabs.logansquare.ConverterUtils; import com.bluelinelabs.logansquare.LoganSquare; import com.nextcloud.talk.R; import com.nextcloud.talk.application.NextcloudTalkApplication; @@ -34,6 +36,7 @@ import com.nextcloud.talk.events.PeerConnectionEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import com.nextcloud.talk.events.WebSocketCommunicationEvent; import com.nextcloud.talk.models.json.signaling.DataChannelMessage; +import com.nextcloud.talk.models.json.signaling.DataChannelMessageNick; import com.nextcloud.talk.models.json.signaling.NCIceCandidate; import com.nextcloud.talk.utils.LoggingUtils; import com.nextcloud.talk.utils.singletons.MerlinTheWizard; @@ -70,6 +73,7 @@ public class MagicPeerConnectionWrapper { private String videoStreamType; private int connectionAttempts = 0; + private PeerConnection.IceConnectionState peerIceConnectionState; @Inject Context context; @@ -163,6 +167,19 @@ public class MagicPeerConnectionWrapper { } } + + public void sendNickChannelData(DataChannelMessageNick dataChannelMessage) { + ByteBuffer buffer; + if (magicDataChannel != null) { + try { + buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes()); + magicDataChannel.send(new DataChannel.Buffer(buffer, false)); + } catch (IOException e) { + Log.d(TAG, "Failed to send channel data, attempting regular " + dataChannelMessage.toString()); + } + } + } + public void sendChannelData(DataChannelMessage dataChannelMessage) { ByteBuffer buffer; if (magicDataChannel != null) { @@ -170,7 +187,7 @@ public class MagicPeerConnectionWrapper { buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes()); magicDataChannel.send(new DataChannel.Buffer(buffer, false)); } catch (IOException e) { - Log.d(TAG, "Failed to send channel data"); + Log.d(TAG, "Failed to send channel data, attempting regular " + dataChannelMessage.toString()); } } } @@ -321,9 +338,11 @@ public class MagicPeerConnectionWrapper { @Override public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { + peerIceConnectionState = iceConnectionState; LoggingUtils.writeLogEntryToFile(context, "iceConnectionChangeTo: " + iceConnectionState.name() + " over " + peerConnection.hashCode() + " " + sessionId); + Log.d("iceConnectionChangeTo: ", iceConnectionState.name() + " over " + peerConnection.hashCode() + " " + sessionId); if (iceConnectionState.equals(PeerConnection.IceConnectionState.CONNECTED)) { connectionAttempts = 0; /*EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType @@ -342,8 +361,11 @@ public class MagicPeerConnectionWrapper { .PEER_CLOSED, sessionId, null, null, videoStreamType)); connectionAttempts = 0; } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.FAILED)) { - if (MerlinTheWizard.isConnectedToInternet() && connectionAttempts < 5) { + /*if (MerlinTheWizard.isConnectedToInternet() && connectionAttempts < 5) { restartIce(); + }*/ + if (isMCUPublisher) { + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED, sessionId, null, null, null)); } } } @@ -453,4 +475,8 @@ public class MagicPeerConnectionWrapper { } } } + + public PeerConnection.IceConnectionState getPeerIceConnectionState() { + return peerIceConnectionState; + } } \ No newline at end of file diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java index 289562f21..c42072239 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebSocketInstance.java @@ -117,11 +117,12 @@ public class MagicWebSocketInstance extends WebSocketListener { } private void closeWebSocket(WebSocket webSocket) { - connected = false; webSocket.close(1000, null); webSocket.cancel(); - messagesQueue = new ArrayList<>(); - currentRoomToken = ""; + if (webSocket == internalWebSocket) { + connected = false; + messagesQueue = new ArrayList<>(); + } } private void restartWebSocket() { @@ -136,8 +137,9 @@ public class MagicWebSocketInstance extends WebSocketListener { @Override public void onMessage(WebSocket webSocket, String text) { + Log.d(TAG, "ReceivingBEFORE : " + webSocket.toString() + " " + text); if (webSocket == internalWebSocket) { - Log.d(TAG, "Receiving : " + text); + Log.d(TAG, "Receiving : " + webSocket.toString() + " " + text); LoggingUtils.writeLogEntryToFile(context, "WebSocket " + webSocket.hashCode() + " receiving: " + text); @@ -163,6 +165,12 @@ public class MagicWebSocketInstance extends WebSocketListener { HashMap helloHasHap = new HashMap<>(); if (!TextUtils.isEmpty(oldResumeId)) { helloHasHap.put("oldResumeId", oldResumeId); + } else { + currentRoomToken = ""; + } + + if (!TextUtils.isEmpty(currentRoomToken)) { + helloHasHap.put("roomToken", currentRoomToken); } eventBus.post(new WebSocketCommunicationEvent("hello", helloHasHap)); break; @@ -172,6 +180,7 @@ public class MagicWebSocketInstance extends WebSocketListener { LoggingUtils.writeLogEntryToFile(context, "WebSocket " + webSocket.hashCode() + " resumeID " + resumeId + " expired"); resumeId = ""; + currentRoomToken = ""; restartWebSocket(); } else if (("hello_expected").equals(errorOverallWebSocketMessage.getErrorWebSocketMessage().getCode())) { restartWebSocket(); @@ -182,11 +191,7 @@ public class MagicWebSocketInstance extends WebSocketListener { JoinedRoomOverallWebSocketMessage joinedRoomOverallWebSocketMessage = LoganSquare.parse(text, JoinedRoomOverallWebSocketMessage.class); currentRoomToken = joinedRoomOverallWebSocketMessage.getRoomWebSocketMessage().getRoomId(); if (joinedRoomOverallWebSocketMessage.getRoomWebSocketMessage().getRoomPropertiesWebSocketMessage() != null && !TextUtils.isEmpty(currentRoomToken)) { - HashMap joinRoomHashMap = new HashMap<>(); - joinRoomHashMap.put("roomToken", currentRoomToken); - eventBus.post(new WebSocketCommunicationEvent("roomJoined", joinRoomHashMap)); - } else { - usersHashMap = new HashMap<>(); + sendRoomJoinedEvent(); } break; case "event": @@ -264,6 +269,12 @@ public class MagicWebSocketInstance extends WebSocketListener { } } + private void sendRoomJoinedEvent() { + HashMap joinRoomHashMap = new HashMap<>(); + joinRoomHashMap.put("roomToken", currentRoomToken); + eventBus.post(new WebSocketCommunicationEvent("roomJoined", joinRoomHashMap)); + } + @Override public void onMessage(WebSocket webSocket, ByteString bytes) { Log.d(TAG, "Receiving bytes : " + bytes.hex()); @@ -272,6 +283,7 @@ public class MagicWebSocketInstance extends WebSocketListener { @Override public void onClosing(WebSocket webSocket, int code, String reason) { Log.d(TAG, "Closing : " + code + " / " + reason); + Log.d("MARIO", String.valueOf(webSocket.hashCode())); LoggingUtils.writeLogEntryToFile(context, "WebSocket " + webSocket.hashCode() + " Closing: " + reason); } @@ -282,7 +294,6 @@ public class MagicWebSocketInstance extends WebSocketListener { LoggingUtils.writeLogEntryToFile(context, "WebSocket " + webSocket.hashCode() + " onFailure: " + t.getMessage()); closeWebSocket(webSocket); - restartWebSocket(); } public String getSessionId() { @@ -299,7 +310,11 @@ public class MagicWebSocketInstance extends WebSocketListener { if (!connected || reconnecting) { messagesQueue.add(message); } else { - internalWebSocket.send(message); + if (roomToken.equals(currentRoomToken)) { + sendRoomJoinedEvent(); + } else { + internalWebSocket.send(message); + } } } catch (IOException e) { e.printStackTrace();