Many improvements

Signed-off-by: Mario Danic <mario@lovelyhq.com>

app/src/main/java/com/nextcloud/talk/activities/CallActivity.java

@@ -49,8 +49,10 @@ import com.nextcloud.talk.api.models.json.signaling.Signaling;
 import com.nextcloud.talk.api.models.json.signaling.SignalingOverall;
 import com.nextcloud.talk.application.NextcloudTalkApplication;
 import com.nextcloud.talk.events.MediaStreamEvent;
+import com.nextcloud.talk.events.PeerReadyEvent;
 import com.nextcloud.talk.events.SessionDescriptionSendEvent;
 import com.nextcloud.talk.persistence.entities.UserEntity;
+import com.nextcloud.talk.webrtc.MagicSdpObserver;
 import com.nextcloud.talk.webrtc.PeerConnectionWrapper;
 
 import org.apache.commons.lang3.StringEscapeUtils;
@@ -268,7 +270,8 @@ public class CallActivity extends AppCompatActivity {
 
         //we already have video and audio tracks. Now create peerconnections
         iceServers = new ArrayList<>();
-        iceServers.add(new PeerConnection.IceServer("stun:stun.nextcloud.com:443"));
+        //iceServers.add(new PeerConnection.IceServer("stun:stun.l.google.com:19302"));
+        iceServers.add(new PeerConnection.IceServer("turn:172.104.225.9:3478"));
 
         //create sdpConstraints
         sdpConstraints = new MediaConstraints();
@@ -306,10 +309,6 @@ public class CallActivity extends AppCompatActivity {
                                                 getPeerConnection();
 
                                         //creating local mediastream
-                                        MediaStream stream = peerConnectionFactory.createLocalMediaStream("NCMS");
-                                        stream.addTrack(localAudioTrack);
-                                        stream.addTrack(localVideoTrack);
-                                        localPeer.addStream(stream);
 
                                         // start pinging the call
                                         ncApi.pingCall(ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()),
@@ -422,17 +421,37 @@ public class CallActivity extends AppCompatActivity {
                 PeerConnectionWrapper peerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId
                         (ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession));
 
-                switch (ncSignalingMessage.getType()) {
+                String type = null;
+                if (ncSignalingMessage.getType() != null) {
+                    type = ncSignalingMessage.getType();
+                } else if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() !=
+                        null) {
+                    type = ncSignalingMessage.getPayload().getType();
+                }
+
+                if (type != null) {
+                    switch (type) {
                         case "offer":
                         case "answer":
                             peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
-                        peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper
+                            peerConnectionWrapper.getPeerConnection().setRemoteDescription(new MagicSdpObserver() {
-                                .getMagicSdpObserver(), new
+
-                                SessionDescription
+                                @Override
-                                (SessionDescription.Type
+                                public void onSetSuccess() {
-                                        .fromCanonicalForm(ncSignalingMessage.getType()),
+                                    super.onSetSuccess();
-                                        ncSignalingMessage
+                                    peerConnectionWrapper.getPeerConnection().createAnswer(new MagicSdpObserver() {
-                                                .getPayload().getSdp()));
+                                        @Override
+                                        public void onCreateSuccess(SessionDescription sessionDescription) {
+                                            super.onCreateSuccess(sessionDescription);
+                                            peerConnectionWrapper.getPeerConnection().setLocalDescription(new MagicSdpObserver(),
+                                                    sessionDescription);
+                                        }
+
+                                    }, sdpConstraints);
+                                }
+                            }, new SessionDescription(SessionDescription.Type.fromCanonicalForm(type),
+                                    ncSignalingMessage.getPayload().getSdp()));
+
 
                             break;
                         case "candidate":
@@ -442,8 +461,17 @@ public class CallActivity extends AppCompatActivity {
                             peerConnectionWrapper.addCandidate(iceCandidate);
                             break;
                         case "endOfCandidates":
-                        Log.d("MARIO", "END CANDIDATES");
                             peerConnectionWrapper.drainIceCandidates();
+                            if (peerConnectionWrapper.getPeerConnection().getLocalDescription() != null) {
+                                EventBus.getDefault().post(new SessionDescriptionSendEvent
+                                        (peerConnectionWrapper.getPeerConnection().getLocalDescription(), callSession,
+                                                peerConnectionWrapper.getPeerConnection().getLocalDescription().type.canonicalForm(),
+                                                null));
+                            } else {
+                                peerConnectionWrapper.getPeerConnection().createAnswer(peerConnectionWrapper
+                                        .getMagicSdpObserver(), sdpConstraints);
+                            }
+                            peerConnectionWrapper.sendLocalCandidates();
                             break;
                         default:
                             break;
@@ -452,6 +480,9 @@ public class CallActivity extends AppCompatActivity {
             } else {
                 Log.d(TAG, "Something went very very wrong");
             }
+        } else {
+            Log.d(TAG, "Something went very very wrong");
+        }
     }
 
     private void processUsersInRoom(List<HashMap<String, String>> users) {
@@ -489,8 +520,10 @@ public class CallActivity extends AppCompatActivity {
                 if (sessionId.compareTo(callSession) < 0) {
                     PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId,
                             false);
+                    if (connectionWrapper.getPeerConnection() != null) {
                         connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(),
                                 sdpConstraints);
+                    }
                 } else {
                     Log.d(TAG, "Waiting for offer");
                 }
@@ -500,19 +533,21 @@ public class CallActivity extends AppCompatActivity {
 
         for (String sessionId : leftSessions) {
             if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
+                if (peerConnectionWrapper.getPeerConnection() != null) {
                     peerConnectionWrapper.getPeerConnection().close();
+                }
                 peerConnectionWrapperList.remove(peerConnectionWrapper);
             }
         }
     }
 
 
+
     private PeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) {
         PeerConnectionWrapper peerConnectionWrapper;
         if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
             return peerConnectionWrapper;
         } else {
-
                 peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
                         iceServers, sdpConstraints, sessionId, isLocalPeer, callSession);
             peerConnectionWrapperList.add(peerConnectionWrapper);
@@ -628,6 +663,16 @@ public class CallActivity extends AppCompatActivity {
         eventBus.unregister(this);
     }
 
+    @Subscribe(threadMode = ThreadMode.BACKGROUND)
+    public void onMessageEvent(PeerReadyEvent peerReadyEvent) {
+        MediaStream stream = peerConnectionFactory.createLocalMediaStream("NCMS");
+        stream.addTrack(localAudioTrack);
+        stream.addTrack(localVideoTrack);
+        if (peerReadyEvent.getPeerConnection() != null) {
+            peerReadyEvent.getPeerConnection().addStream(stream);
+        }
+    }
+
     @Subscribe(threadMode = ThreadMode.BACKGROUND)
     public void onMessageEvent(MediaStreamEvent mediaStreamEvent) {
         gotRemoteStream(mediaStreamEvent.getMediaStream());
@@ -660,38 +705,47 @@ public class CallActivity extends AppCompatActivity {
 
 
         StringBuilder stringBuilder = new StringBuilder();
-        stringBuilder.append("[");
         stringBuilder.append("{");
         stringBuilder.append("\"fn\":\"");
-        stringBuilder.append(StringEscapeUtils.escapeEcmaScript(LoganSquare.serialize(ncMessageWrapper
+        stringBuilder.append(StringEscapeUtils.escapeJson(LoganSquare.serialize(ncMessageWrapper
-                .getSignalingMessage())) + "\"");
+                .getSignalingMessage()))).append("\"");
         stringBuilder.append(",");
         stringBuilder.append("\"sessionId\":");
-        stringBuilder.append("\"").append(callSession).append("\"");
+        stringBuilder.append("\"").append(StringEscapeUtils.escapeJson(callSession)).append("\"");
         stringBuilder.append(",");
         stringBuilder.append("\"ev\":\"message\"");
         stringBuilder.append("}");
-        stringBuilder.append("]");
 
+        List<String> strings = new ArrayList<>();
         String stringToSend = stringBuilder.toString();
-        Log.d("MARIO", stringToSend);
+        strings.add(stringToSend);
+
+        List<NCMessageWrapper> wrappers = new ArrayList<>();
+        wrappers.add(ncMessageWrapper);
         ncApi.sendSignalingMessages(credentials, ApiHelper.getUrlForSignaling(userEntity.getBaseUrl()),
-                    stringToSend)
+                strings.toString())
                     .subscribeOn(Schedulers.newThread())
-                    .subscribe(new Observer<GenericOverall>() {
+                    .subscribe(new Observer<SignalingOverall>() {
                         @Override
                         public void onSubscribe(Disposable d) {
 
                         }
 
                         @Override
-                        public void onNext(GenericOverall genericOverall) {
+                        public void onNext(SignalingOverall signalingOverall) {
-
+                            if (signalingOverall.getOcs().getSignalings() != null) {
+                                for (int i = 0; i < signalingOverall.getOcs().getSignalings().size(); i++) {
+                                    try {
+                                        receivedSignalingMessage(signalingOverall.getOcs().getSignalings().get(i));
+                                    } catch (IOException e) {
+                                        e.printStackTrace();
+                                    }
+                                }
+                            }
                         }
 
                         @Override
                         public void onError(Throwable e) {
-
                         }
 
                         @Override
@@ -701,5 +755,4 @@ public class CallActivity extends AppCompatActivity {
                     });
     }
 
-
 }

app/src/main/java/com/nextcloud/talk/api/NcApi.java

@@ -35,8 +35,8 @@ import com.nextcloud.talk.api.models.json.userprofile.UserProfileOverall;
 import java.util.Map;
 
 import io.reactivex.Observable;
-import retrofit2.http.Body;
 import retrofit2.http.DELETE;
+import retrofit2.http.Field;
 import retrofit2.http.FieldMap;
 import retrofit2.http.FormUrlEncoded;
 import retrofit2.http.GET;
@@ -165,9 +165,10 @@ public interface NcApi {
 
         Server URL is: baseUrl + ocsApiVersion + spreedApiVersion + /signaling
     */
+    @FormUrlEncoded
     @POST
-    Observable<GenericOverall> sendSignalingMessages(@Header("Authorization") String authorization, @Url String url,
+    Observable<SignalingOverall> sendSignalingMessages(@Header("Authorization") String authorization, @Url String url,
-                                                     @Body String message);
+                                             @Field("messages") String messages);
 
     /*
         Server URL is: baseUrl + ocsApiVersion + spreedApiVersion + /signaling

app/src/main/java/com/nextcloud/talk/events/PeerReadyEvent.java

@@ -0,0 +1,35 @@
+/*
+ * Nextcloud Talk application
+ *
+ * @author Mario Danic
+ * Copyright (C) 2017 Mario Danic <mario@lovelyhq.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.nextcloud.talk.events;
+
+
+import org.webrtc.PeerConnection;
+
+import lombok.Data;
+
+@Data
+public class PeerReadyEvent {
+    private final PeerConnection peerConnection;
+
+    public PeerReadyEvent(PeerConnection peerConnection) {
+        this.peerConnection = peerConnection;
+    }
+}

app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java

@@ -55,8 +55,6 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer {
 
     @Override
     public void onIceCandidate(IceCandidate iceCandidate) {
-
-
     }
 
     @Override

app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java

@@ -25,6 +25,7 @@ import android.util.Log;
 import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage;
 import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate;
 import com.nextcloud.talk.events.MediaStreamEvent;
+import com.nextcloud.talk.events.PeerReadyEvent;
 import com.nextcloud.talk.events.SessionDescriptionSendEvent;
 
 import org.greenrobot.eventbus.EventBus;
@@ -79,6 +80,7 @@ public class PeerConnectionWrapper {
 
             @Override
             public void onAddStream(MediaStream mediaStream) {
+                Log.d("MARIO", "MEDIA");
                 EventBus.getDefault().post(new MediaStreamEvent(mediaStream));
             }
 
@@ -90,9 +92,9 @@ public class PeerConnectionWrapper {
 
             @Override
             public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
-                if (iceGatheringState.equals(PeerConnection.IceGatheringState.COMPLETE)) {
+                /*if (iceGatheringState.equals(PeerConnection.IceGatheringState.COMPLETE)) {
                     sendLocalCandidates();
-                }
+                }*/
             }
 
             @Override
@@ -108,6 +110,11 @@ public class PeerConnectionWrapper {
 
                 peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints,
                         magicPeerConnectionObserver);
+
+
+        if (isLocalPeer) {
+            EventBus.getDefault().post(new PeerReadyEvent(peerConnection));
+        }
         this.sessionId = sessionId;
         this.local = isLocalPeer;
         this.mediaConstraints = mediaConstraints;
@@ -139,11 +146,7 @@ public class PeerConnectionWrapper {
                     if (peerConnection.getRemoteDescription() == null) {
                         // We've just set our local SDP so time to send it.
                         EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId,
-                                "offer", null));
+                                peerConnection.getLocalDescription().type.canonicalForm(), null));
-                    } else {
-                        // We've just set remote description, so drain remote
-                        // and send local ICE candidates.
-                        drainIceCandidates();
                     }
                 } else {
                     // For anwering peer connection we set remote SDP and then
@@ -152,18 +155,13 @@ public class PeerConnectionWrapper {
                         EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription
                                 (),
                                 sessionId,
-                                "answer", null));
+                                peerConnection.getLocalDescription().type.canonicalForm(), null));
                         // We've just set our local SDP so time to send it, drain
                         // remote and send local ICE candidates.
 
                     } else {
-                        peerConnection.createAnswer(magicSdpObserver, mediaConstraints);
-
                         // We've just set remote SDP - do nothing for now -
                         // answer will be created soon.
-                        /*Log.d("MARIO", "CREATING ANSWER");
-                        drainIceCandidates();
-                        peerConnection.createAnswer(magicSdpObserver, mediaConstraints);*/
 
                     }
                 }
@@ -174,11 +172,13 @@ public class PeerConnectionWrapper {
     }
 
 
-    private void sendLocalCandidates() {
+    public void sendLocalCandidates() {
         for (NCIceCandidate ncIceCandidate : localCandidates) {
             EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
                     "candidate", ncIceCandidate));
         }
+
+        localCandidates = new ArrayList<>();
     }
 
     public void drainIceCandidates() {