diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index fb565cb9c..7444864e7 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -34,6 +34,8 @@ import android.util.TypedValue; import android.view.View; import android.view.Window; import android.view.WindowManager; +import android.widget.GridLayout; +import android.widget.RelativeLayout; import com.bluelinelabs.logansquare.LoganSquare; import com.nextcloud.talk.R; @@ -52,7 +54,7 @@ import com.nextcloud.talk.events.MediaStreamEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import com.nextcloud.talk.persistence.entities.UserEntity; import com.nextcloud.talk.webrtc.MagicAudioManager; -import com.nextcloud.talk.webrtc.PeerConnectionWrapper; +import com.nextcloud.talk.webrtc.MagicPeerConnectionWrapper; import org.apache.commons.lang3.StringEscapeUtils; import org.greenrobot.eventbus.EventBus; @@ -104,8 +106,8 @@ public class CallActivity extends AppCompatActivity { @BindView(R.id.pip_video_view) SurfaceViewRenderer pipVideoView; - @BindView(R.id.fullscreen_video_view) - SurfaceViewRenderer fullScreenVideoView; + @BindView(R.id.videos_grid_view) + GridLayout videosGrid; @Inject NcApi ncApi; @@ -125,7 +127,9 @@ public class CallActivity extends AppCompatActivity { VideoCapturer videoCapturer; VideoRenderer localRenderer; VideoRenderer remoteRenderer; + HashMap videoRendererHashMap = new HashMap<>(); PeerConnection localPeer; + EglBase rootEglBase; boolean leavingCall = false; BooleanSupplier booleanSupplier = () -> leavingCall; Disposable signalingDisposable; @@ -138,7 +142,7 @@ public class CallActivity extends AppCompatActivity { private MediaStream localMediaStream; private String credentials; - private List peerConnectionWrapperList = new ArrayList<>(); + private List magicPeerConnectionWrapperList = new ArrayList<>(); private static int getSystemUiVisibility() { int flags = View.SYSTEM_UI_FLAG_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_FULLSCREEN; @@ -224,16 +228,11 @@ public class CallActivity extends AppCompatActivity { public void initViews() { pipVideoView.setMirror(true); - fullScreenVideoView.setMirror(false); - EglBase rootEglBase = EglBase.create(); + rootEglBase = EglBase.create(); pipVideoView.init(rootEglBase.getEglBaseContext(), null); pipVideoView.setZOrderMediaOverlay(true); - fullScreenVideoView.init(rootEglBase.getEglBaseContext(), null); - fullScreenVideoView.setZOrderMediaOverlay(true); - fullScreenVideoView.setEnableHardwareScaler(true); pipVideoView.setEnableHardwareScaler(true); pipVideoView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT); - fullScreenVideoView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); } public void start() { @@ -435,7 +434,7 @@ public class CallActivity extends AppCompatActivity { NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(), NCSignalingMessage.class); if (ncSignalingMessage.getRoomType().equals("video")) { - PeerConnectionWrapper peerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId + MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId (ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession)); String type = null; @@ -450,8 +449,8 @@ public class CallActivity extends AppCompatActivity { switch (type) { case "offer": case "answer": - peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper + magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); + magicPeerConnectionWrapper.getPeerConnection().setRemoteDescription(magicPeerConnectionWrapper .getMagicSdpObserver(), new SessionDescription(SessionDescription.Type.fromCanonicalForm(type), ncSignalingMessage.getPayload().getSdp())); break; @@ -459,11 +458,11 @@ public class CallActivity extends AppCompatActivity { NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate()); - peerConnectionWrapper.addCandidate(iceCandidate); + magicPeerConnectionWrapper.addCandidate(iceCandidate); break; case "endOfCandidates": - peerConnectionWrapper.drainIceCandidates(); - peerConnectionWrapper.sendLocalCandidates(); + magicPeerConnectionWrapper.drainIceCandidates(); + magicPeerConnectionWrapper.sendLocalCandidates(); break; default: break; @@ -496,9 +495,9 @@ public class CallActivity extends AppCompatActivity { } } - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (!peerConnectionWrapper.getSessionId().equals(callSession)) { - oldSesssions.add(peerConnectionWrapper.getSessionId()); + for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) { + if (!magicPeerConnectionWrapper.getSessionId().equals(callSession)) { + oldSesssions.add(magicPeerConnectionWrapper.getSessionId()); } } @@ -513,12 +512,12 @@ public class CallActivity extends AppCompatActivity { return; } - PeerConnectionWrapper peerConnectionWrapper; + MagicPeerConnectionWrapper magicPeerConnectionWrapper; for (String sessionId : newSessions) { if (getPeerConnectionWrapperForSessionId(sessionId) == null) { if (sessionId.compareTo(callSession) < 0) { - PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId, + MagicPeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId, false); if (connectionWrapper.getPeerConnection() != null) { connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(), @@ -532,33 +531,33 @@ public class CallActivity extends AppCompatActivity { } for (String sessionId : leftSessions) { - if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { - if (peerConnectionWrapper.getPeerConnection() != null) { - peerConnectionWrapper.getPeerConnection().close(); + if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { + if (magicPeerConnectionWrapper.getPeerConnection() != null) { + magicPeerConnectionWrapper.getPeerConnection().close(); } - peerConnectionWrapperList.remove(peerConnectionWrapper); + magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper); } } } - private PeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) { - PeerConnectionWrapper peerConnectionWrapper; - if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { - return peerConnectionWrapper; + private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) { + MagicPeerConnectionWrapper magicPeerConnectionWrapper; + if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { + return magicPeerConnectionWrapper; } else { - peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, + magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, iceServers, sdpConstraints, sessionId); - peerConnectionWrapper.getPeerConnection().addStream(localMediaStream); - peerConnectionWrapperList.add(peerConnectionWrapper); - return peerConnectionWrapper; + magicPeerConnectionWrapper.getPeerConnection().addStream(localMediaStream); + magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper); + return magicPeerConnectionWrapper; } } - private PeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) { - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - return peerConnectionWrapper; + private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) { + for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) { + if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) { + return magicPeerConnectionWrapper; } } return null; @@ -570,9 +569,9 @@ public class CallActivity extends AppCompatActivity { dispose(null); - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (peerConnectionWrapper.getPeerConnection() != null) { - peerConnectionWrapper.getPeerConnection().close(); + for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) { + if (magicPeerConnectionWrapper.getPeerConnection() != null) { + magicPeerConnectionWrapper.getPeerConnection().close(); } } @@ -581,7 +580,6 @@ public class CallActivity extends AppCompatActivity { } pipVideoView.release(); - fullScreenVideoView.release(); String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()); ncApi.leaveCall(credentials, ApiHelper.getUrlForCall(userEntity.getBaseUrl(), roomToken)) @@ -610,7 +608,7 @@ public class CallActivity extends AppCompatActivity { }); } - private void gotRemoteStream(MediaStream stream) { + private void gotRemoteStream(MediaStream stream, String session) { //we have remote video stream. add to the renderer. if (stream.videoTracks.size() < 2 && stream.audioTracks.size() < 2) { if (stream.videoTracks.size() == 1) { @@ -621,8 +619,26 @@ public class CallActivity extends AppCompatActivity { public void run() { if (stream.videoTracks.size() == 1) { try { - remoteRenderer = new VideoRenderer(fullScreenVideoView); + RelativeLayout relativeLayout = (RelativeLayout) + getLayoutInflater().inflate(R.layout.surface_renderer, videosGrid, + false); + relativeLayout.setTag(session); + SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id + .surface_view); + surfaceViewRenderer.setMirror(false); + surfaceViewRenderer.init(rootEglBase.getEglBaseContext(), null); + surfaceViewRenderer.setZOrderMediaOverlay(true); + surfaceViewRenderer.setEnableHardwareScaler(true); + surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); + VideoRenderer remoteRenderer = new VideoRenderer(surfaceViewRenderer); + videoRendererHashMap.put(session, remoteRenderer); videoTrack.addRenderer(remoteRenderer); + videosGrid.addView(relativeLayout); + + GridLayout.LayoutParams param = new GridLayout.LayoutParams(GridLayout.spec( + GridLayout.UNDEFINED,GridLayout.FILL,1f), + GridLayout.spec(GridLayout.UNDEFINED,GridLayout.FILL,1f)); + relativeLayout.setLayoutParams(param); } catch (Exception e) { e.printStackTrace(); } @@ -676,7 +692,7 @@ public class CallActivity extends AppCompatActivity { @Subscribe(threadMode = ThreadMode.BACKGROUND) public void onMessageEvent(MediaStreamEvent mediaStreamEvent) { - gotRemoteStream(mediaStreamEvent.getMediaStream()); + gotRemoteStream(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession()); } @Subscribe(threadMode = ThreadMode.BACKGROUND) diff --git a/app/src/main/java/com/nextcloud/talk/api/models/json/signaling/DataChannelMessage.java b/app/src/main/java/com/nextcloud/talk/api/models/json/signaling/DataChannelMessage.java index 1ce1bab7d..b29214853 100644 --- a/app/src/main/java/com/nextcloud/talk/api/models/json/signaling/DataChannelMessage.java +++ b/app/src/main/java/com/nextcloud/talk/api/models/json/signaling/DataChannelMessage.java @@ -33,4 +33,11 @@ public class DataChannelMessage { @JsonField(name = "payload") String payload; + + public DataChannelMessage(String type) { + this.type = type; + } + + public DataChannelMessage() { + } } diff --git a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java index 34b598198..8fd8f5762 100644 --- a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java @@ -27,8 +27,10 @@ import lombok.Data; @Data public class MediaStreamEvent { private final MediaStream mediaStream; + private final String session; - public MediaStreamEvent(MediaStream mediaStream) { + public MediaStreamEvent(MediaStream mediaStream, String session) { this.mediaStream = mediaStream; + this.session = session; } } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java index 940d7303b..d1cb7c786 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java @@ -20,8 +20,6 @@ package com.nextcloud.talk.webrtc; -import android.util.Log; - import org.webrtc.DataChannel; import org.webrtc.IceCandidate; import org.webrtc.MediaStream; @@ -35,7 +33,6 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { - Log.d("MARIO", signalingState.name()); } @Override diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java similarity index 72% rename from app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java rename to app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java index 703bb05fe..957e10a23 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java @@ -22,6 +22,7 @@ package com.nextcloud.talk.webrtc; import android.util.Log; +import com.bluelinelabs.logansquare.LoganSquare; import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage; import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate; import com.nextcloud.talk.events.MediaStreamEvent; @@ -36,12 +37,13 @@ import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.SessionDescription; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -public class PeerConnectionWrapper { - private static String TAG = "PeerConnectionWrapper"; +public class MagicPeerConnectionWrapper { + private static String TAG = "MagicPeerConnectionWrapper"; private static PeerConnection peerConnection; List iceCandidates = new ArrayList<>(); List iceServers; @@ -49,39 +51,21 @@ public class PeerConnectionWrapper { private String sessionId; private String nick; private MediaConstraints mediaConstraints; - private DataChannel dataChannel; + private DataChannel magicDataChannel; private MagicSdpObserver magicSdpObserver; private MagicPeerConnectionObserver magicPeerConnectionObserver; - public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, - List iceServerList, - MediaConstraints mediaConstraints, - String sessionId) { + public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, + List iceServerList, + MediaConstraints mediaConstraints, + String sessionId) { this.iceServers = iceServerList; magicPeerConnectionObserver = new MagicPeerConnectionObserver() { - @Override - public void onIceConnectionReceivingChange(boolean b) { - - } - - @Override - public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { - } - @Override public void onAddStream(MediaStream mediaStream) { - EventBus.getDefault().post(new MediaStreamEvent(mediaStream)); - } - - @Override - public void onSignalingChange(PeerConnection.SignalingState signalingState) { - } - - - @Override - public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) { + EventBus.getDefault().post(new MediaStreamEvent(mediaStream, sessionId)); } @Override @@ -90,12 +74,15 @@ public class PeerConnectionWrapper { ncIceCandidate.setSdpMid(iceCandidate.sdpMid); ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex); ncIceCandidate.setCandidate(iceCandidate.sdp); - if (peerConnection.getRemoteDescription() == null) { + /*if (peerConnection.getRemoteDescription() == null) { localCandidates.add(ncIceCandidate); } else { EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId, "candidate", ncIceCandidate)); - } + }*/ + EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId, + "candidate", ncIceCandidate)); + } }; @@ -103,6 +90,11 @@ public class PeerConnectionWrapper { peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints, magicPeerConnectionObserver); + DataChannel.Init init = new DataChannel.Init(); + init.negotiated = false; + magicDataChannel = peerConnection.createDataChannel("status", init); + magicDataChannel.registerObserver(new MagicDataChannelObserver()); + this.sessionId = sessionId; this.mediaConstraints = mediaConstraints; @@ -120,6 +112,7 @@ public class PeerConnectionWrapper { @Override public void onCreateSuccess(SessionDescription sessionDescription) { super.onCreateSuccess(sessionDescription); + peerConnection.setLocalDescription(magicSdpObserver, sessionDescription); } @@ -181,8 +174,13 @@ public class PeerConnectionWrapper { } private void sendChannelData(DataChannelMessage dataChannelMessage) { - ByteBuffer buffer = ByteBuffer.wrap(dataChannelMessage.toString().getBytes()); - dataChannel.send(new DataChannel.Buffer(buffer, false)); + ByteBuffer buffer = null; + try { + buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes()); + magicDataChannel.send(new DataChannel.Buffer(buffer, false)); + } catch (IOException e) { + e.printStackTrace(); + } } @@ -191,7 +189,7 @@ public class PeerConnectionWrapper { } public static void setPeerConnection(PeerConnection peerConnection) { - PeerConnectionWrapper.peerConnection = peerConnection; + MagicPeerConnectionWrapper.peerConnection = peerConnection; } public String getSessionId() { @@ -209,4 +207,35 @@ public class PeerConnectionWrapper { public void setNick(String nick) { this.nick = nick; } + + private class MagicDataChannelObserver implements DataChannel.Observer { + + @Override + public void onBufferedAmountChange(long l) { + + } + + @Override + public void onStateChange() { + if (magicDataChannel.state().equals(DataChannel.State.OPEN) && + magicDataChannel.label().equals("status")) { + sendChannelData(new DataChannelMessage("videoOn")); + sendChannelData(new DataChannelMessage("audioOn")); + } + + } + + @Override + public void onMessage(DataChannel.Buffer buffer) { + if (buffer.binary) { + Log.d(TAG, "Received binary msg over " + TAG + " " + sessionId); + return; + } + ByteBuffer data = buffer.data; + final byte[] bytes = new byte[data.capacity()]; + data.get(bytes); + String strData = new String(bytes); + Log.d(TAG, "Got msg: " + strData + " over " + TAG + " " + sessionId); + } + } } diff --git a/app/src/main/res/layout/activity_call.xml b/app/src/main/res/layout/activity_call.xml index 33c31309c..9d0722540 100644 --- a/app/src/main/res/layout/activity_call.xml +++ b/app/src/main/res/layout/activity_call.xml @@ -26,6 +26,15 @@ android:fitsSystemWindows="true" tools:context=".activities.CallActivity"> + + + + - - \ No newline at end of file diff --git a/app/src/main/res/layout/surface_renderer.xml b/app/src/main/res/layout/surface_renderer.xml new file mode 100644 index 000000000..a3569fa63 --- /dev/null +++ b/app/src/main/res/layout/surface_renderer.xml @@ -0,0 +1,33 @@ + + + + + + + + \ No newline at end of file