diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 9ec24cc42..241469d8c 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -48,6 +48,7 @@ import com.nextcloud.talk.api.models.json.signaling.NCSignalingMessage; import com.nextcloud.talk.api.models.json.signaling.Signaling; import com.nextcloud.talk.api.models.json.signaling.SignalingOverall; import com.nextcloud.talk.application.NextcloudTalkApplication; +import com.nextcloud.talk.events.MediaStreamEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import com.nextcloud.talk.persistence.entities.UserEntity; import com.nextcloud.talk.webrtc.PeerConnectionWrapper; @@ -55,6 +56,7 @@ import com.nextcloud.talk.webrtc.PeerConnectionWrapper; import org.greenrobot.eventbus.EventBus; import org.greenrobot.eventbus.Subscribe; import org.greenrobot.eventbus.ThreadMode; +import org.parceler.Parcels; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; import org.webrtc.Camera1Enumerator; @@ -154,8 +156,8 @@ public class CallActivity extends AppCompatActivity { ButterKnife.bind(this); roomToken = getIntent().getExtras().getString("roomToken", ""); - userEntity = getIntent().getExtras().getParcelable("userEntity"); - callSession = getIntent().getExtras().getString("callSession", ""); + userEntity = Parcels.unwrap(getIntent().getExtras().getParcelable("userEntity")); + callSession = "0"; credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()); initViews(); @@ -274,14 +276,6 @@ public class CallActivity extends AppCompatActivity { sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true")); sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); - PeerConnection localPeer = alwaysGetPeerConnectionWrapperForSessionId(callSession, true). - getPeerConnection(); - - //creating local mediastream - MediaStream stream = peerConnectionFactory.createLocalMediaStream("NCMS"); - stream.addTrack(localAudioTrack); - stream.addTrack(localVideoTrack); - localPeer.addStream(stream); ncApi.joinRoom(credentials, ApiHelper.getUrlForJoinRoom(userEntity.getBaseUrl(), roomToken)) .subscribeOn(Schedulers.newThread()) @@ -307,6 +301,14 @@ public class CallActivity extends AppCompatActivity { @Override public void onNext(GenericOverall genericOverall) { callSession = callOverall.getOcs().getData().getSessionId(); + localPeer = alwaysGetPeerConnectionWrapperForSessionId(callSession, true). + getPeerConnection(); + + //creating local mediastream + MediaStream stream = peerConnectionFactory.createLocalMediaStream("NCMS"); + stream.addTrack(localAudioTrack); + stream.addTrack(localVideoTrack); + localPeer.addStream(stream); // start pinging the call ncApi.pingCall(ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()), @@ -423,14 +425,14 @@ public class CallActivity extends AppCompatActivity { case "offer": case "answer": peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper.getMagicSdpObserver(), - new SessionDescription(SessionDescription.Type.fromCanonicalForm(ncSignalingMessage.getType()), + peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper + .getMagicSdpObserver(), new + SessionDescription + (SessionDescription.Type + .fromCanonicalForm(ncSignalingMessage.getType()), ncSignalingMessage - .getPayload().getSdp())); + .getPayload().getSdp())); - /*if (ncSignalingMessage.getType().equals("offer")) { - peerConnectionWrapper.sendMessage(true); - }*/ break; case "candidate": NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); @@ -438,6 +440,10 @@ public class CallActivity extends AppCompatActivity { ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate()); peerConnectionWrapper.addCandidate(iceCandidate); break; + case "endOfCandidates": + Log.d("MARIO", "END CANDIDATES"); + peerConnectionWrapper.drainIceCandidates(); + break; default: break; } @@ -482,7 +488,7 @@ public class CallActivity extends AppCompatActivity { if (sessionId.compareTo(callSession) < 0 ) { PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId, false); - connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(), sdpConstraints); + connectionWrapper.getPeerConnection().createAnswer(connectionWrapper.getMagicSdpObserver(), sdpConstraints); } else { Log.d(TAG, "Waiting for offer"); } @@ -585,20 +591,6 @@ public class CallActivity extends AppCompatActivity { } - public void onIceCandidateReceived(boolean isLocalPeer, IceCandidate iceCandidate) { - //we have received ice candidate. We can set it to the other peer. - if (!isLocalPeer) { - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (!peerConnectionWrapper.isLocal()) { - //peerConnectionWrapper.addCandidate(iceCandidate); - } - } - } else { - alwaysGetPeerConnectionWrapperForSessionId(callSession, true).getPeerConnection().addIceCandidate - (iceCandidate); - } - } - @Override public void onDestroy() { super.onDestroy(); @@ -634,9 +626,14 @@ public class CallActivity extends AppCompatActivity { eventBus.unregister(this); } + @Subscribe(threadMode = ThreadMode.BACKGROUND) + public void onMessageEvent(MediaStreamEvent mediaStreamEvent) { + gotRemoteStream(mediaStreamEvent.getMediaStream()); + } + @Subscribe(threadMode = ThreadMode.BACKGROUND) public void onMessageEvent(SessionDescriptionSendEvent sessionDescriptionSend) { - Log.d("MARIO", "SENDING " + sessionDescriptionSend.getType()); + Log.d("MARIO_123", "SENDING " + sessionDescriptionSend.getType()); String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()); NCMessageWrapper ncMessageWrapper = new NCMessageWrapper(); ncMessageWrapper.setEv("message"); diff --git a/app/src/main/java/com/nextcloud/talk/controllers/CallsListController.java b/app/src/main/java/com/nextcloud/talk/controllers/CallsListController.java index 1839b75c2..c531319d9 100644 --- a/app/src/main/java/com/nextcloud/talk/controllers/CallsListController.java +++ b/app/src/main/java/com/nextcloud/talk/controllers/CallsListController.java @@ -60,6 +60,8 @@ import com.nextcloud.talk.persistence.entities.UserEntity; import com.nextcloud.talk.utils.bundle.BundleBuilder; import com.nextcloud.talk.utils.database.user.UserUtils; +import org.parceler.Parcels; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -113,7 +115,7 @@ public class CallsListController extends BaseController implements SearchView.On Intent callIntent = new Intent(getActivity(), CallActivity.class); BundleBuilder bundleBuilder = new BundleBuilder(new Bundle()); bundleBuilder.putString("roomToken", roomItem.getModel().getToken()); - bundleBuilder.putParcelable("userEntity", userEntity); + bundleBuilder.putParcelable("userEntity", Parcels.wrap(userEntity)); callIntent.putExtras(bundleBuilder.build()); startActivity(callIntent); } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 9f86fd93f..fff9efecb 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -43,6 +43,9 @@ import java.util.List; public class PeerConnectionWrapper { private static String TAG = "PeerConnectionWrapper"; private static PeerConnection peerConnection; + List iceCandidates = new ArrayList<>(); + List iceServers; + List candidatesToSend = new ArrayList<>(); private String sessionId; private String callToken; private String nick; @@ -51,17 +54,31 @@ public class PeerConnectionWrapper { private DataChannel dataChannel; private MagicSdpObserver magicSdpObserver; private MagicPeerConnectionObserver magicPeerConnectionObserver; - List iceCandidates = new ArrayList<>(); + private boolean isInitiator; + List sessionDescriptionsQueue = new ArrayList<>(); + List localCandidates = new ArrayList<>(); + List sessionDescriptionSendEvents = new ArrayList<>(); public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, List iceServerList, MediaConstraints mediaConstraints, String sessionId, boolean isLocalPeer, String callToken) { + this.iceServers = iceServerList; + magicPeerConnectionObserver = new MagicPeerConnectionObserver() { + @Override + public void onIceConnectionReceivingChange(boolean b) { + + } + + @Override + public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { + Log.d("MARIO_ICE", iceConnectionState.name()); + } + @Override public void onAddStream(MediaStream mediaStream) { - Log.d("MARIO", "MEDIA STREAM"); EventBus.getDefault().post(new MediaStreamEvent(mediaStream)); } @@ -70,15 +87,19 @@ public class PeerConnectionWrapper { Log.d("MARIO", signalingState.name()); } + + @Override + public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) { + } + @Override public void onIceCandidate(IceCandidate iceCandidate) { - NCIceCandidate ncIceCandidate = new NCIceCandidate(); - ncIceCandidate.setCandidate(iceCandidate.sdp); - ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex); - ncIceCandidate.setSdpMid(iceCandidate.sdpMid); - EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId, "candidate", - ncIceCandidate)); - Log.d("MARIO", "SENDING REMOTE CANDIDATE"); + Log.d("MARIO_GENERT", "BLA"); + NCIceCandidate ncIceCandidate = new NCIceCandidate(); + ncIceCandidate.setSdpMid(iceCandidate.sdpMid); + ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex); + ncIceCandidate.setCandidate(iceCandidate.sdp); + localCandidates.add(ncIceCandidate); } }; @@ -89,7 +110,7 @@ public class PeerConnectionWrapper { this.local = isLocalPeer; this.mediaConstraints = mediaConstraints; this.callToken = callToken; - boolean isInitiator = this.sessionId.compareTo(callToken) < 0; + isInitiator = this.sessionId.compareTo(callToken) < 0; magicSdpObserver = new MagicSdpObserver() { @Override @@ -120,24 +141,38 @@ public class PeerConnectionWrapper { } else { // We've just set remote description, so drain remote // and send local ICE candidates. - Log.d("MARIO", "DRAINING CANDIDATES"); drainIceCandidates(); } } else { - // For answering peer connection we set remote SDP and then + // For anwering peer connection we set remote SDP and then // create answer and set local SDP. if (peerConnection.getLocalDescription() != null) { // We've just set our local SDP so time to send it, drain // remote and send local ICE candidates. Log.d("MARIO", "SENDING ANSWER FROM OBSERVER"); - EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, + sendLocalCandidates(); + EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription + (), + sessionId, "answer", null)); - drainIceCandidates(); + } else { + peerConnection.createAnswer(new MagicSdpObserver() { + @Override + public void onCreateSuccess(SessionDescription sessionDescription) { + super.onCreateSuccess(sessionDescription); + peerConnection.setLocalDescription(magicSdpObserver, sessionDescription); + sendLocalCandidates(); + } + + }, mediaConstraints); + // We've just set remote SDP - do nothing for now - // answer will be created soon. - Log.d("MARIO", "CREATING ANSWER"); - peerConnection.createAnswer(magicSdpObserver, mediaConstraints); + /*Log.d("MARIO", "CREATING ANSWER"); + drainIceCandidates(); + peerConnection.createAnswer(magicSdpObserver, mediaConstraints);*/ + } } } @@ -146,12 +181,25 @@ public class PeerConnectionWrapper { } - private void drainIceCandidates() { + + + private void sendLocalCandidates() { + Log.d("MARIO", "SENDING LOCAL"); + for (NCIceCandidate ncIceCandidate : localCandidates) { + EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId, + "candidate", ncIceCandidate)); + } + } + + public void drainIceCandidates() { + Log.d("MARIO", "DRAINING"); + for (IceCandidate iceCandidate : iceCandidates) { peerConnection.addIceCandidate(iceCandidate); } iceCandidates = new ArrayList<>(); + } public MagicSdpObserver getMagicSdpObserver() { @@ -159,77 +207,22 @@ public class PeerConnectionWrapper { } public void addCandidate(IceCandidate iceCandidate) { + Log.d("MARIO", "RECEIVING CANDIDATE"); if (peerConnection.getRemoteDescription() != null) { - // queue + Log.d("MARIO", "DIRECT ADDING"); peerConnection.addIceCandidate(iceCandidate); } else { + Log.d("MARIO", "DIRECT QUEUE"); iceCandidates.add(iceCandidate); } } - public void sendMessage(boolean isAnswer) { - - if (!isAnswer) { - peerConnection.createOffer(magicSdpObserver, mediaConstraints); - } else { - peerConnection.createAnswer(magicSdpObserver, mediaConstraints); - } - } - private void sendChannelData(DataChannelMessage dataChannelMessage) { ByteBuffer buffer = ByteBuffer.wrap(dataChannelMessage.toString().getBytes()); dataChannel.send(new DataChannel.Buffer(buffer, false)); } - public void sendOffer() { - DataChannel.Init dcInit = new DataChannel.Init(); - dcInit.negotiated = false; - dataChannel = peerConnection.createDataChannel("status", dcInit); - dataChannel.registerObserver(new DataChannel.Observer() { - @Override - public void onBufferedAmountChange(long l) { - - } - - @Override - public void onStateChange() { - if (dataChannel.state() == DataChannel.State.OPEN && dataChannel.label().equals("status")) { - DataChannelMessage dataChannelMessage = new DataChannelMessage(); - dataChannelMessage.setType("videoOn"); - sendChannelData(dataChannelMessage); - dataChannelMessage.setType("audioOn"); - sendChannelData(dataChannelMessage); - } - - } - - @Override - public void onMessage(DataChannel.Buffer buffer) { - ByteBuffer data = buffer.data; - byte[] bytes = new byte[data.remaining()]; - data.get(bytes); - final String command = new String(bytes); - } - }); - - MagicSdpObserver magicSdpObserver = new MagicSdpObserver() { - public void onCreateSuccess(SessionDescription sessionDescription) { - super.onCreateSuccess(sessionDescription); - peerConnection.setLocalDescription(new MagicSdpObserver(), sessionDescription); - } - - @Override - public void onSetSuccess() { - EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, - "offer", null)); - } - - }; - - peerConnection.createOffer(magicSdpObserver, mediaConstraints); - } - public boolean isLocal() { return local; }