gronod/talk-android
1
0
Fork 1
mirror of https://github.com/nextcloud/talk-android synced 2025-06-19 11:39:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

Progress

Browse Source 
Signed-off-by: Mario Danic <mario@lovelyhq.com>
This commit is contained in:
Mario Danic 2017-11-22 15:23:28 +01:00
parent 8eee14d32f
commit 855ed05cf6
7 changed files with 172 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
app/src/main/java/com/nextcloud/talk/activities/CallActivity.java
View File

@ -34,6 +34,8 @@ import android.util.TypedValue;
import android.view.View;
import android.view.Window;
import android.view.WindowManager;
import android.widget.GridLayout;
import android.widget.RelativeLayout;
import com.bluelinelabs.logansquare.LoganSquare;
import com.nextcloud.talk.R;
@ -52,7 +54,7 @@ import com.nextcloud.talk.events.MediaStreamEvent;
import com.nextcloud.talk.events.SessionDescriptionSendEvent;
import com.nextcloud.talk.persistence.entities.UserEntity;
import com.nextcloud.talk.webrtc.MagicAudioManager;
import com.nextcloud.talk.webrtc.PeerConnectionWrapper;
import com.nextcloud.talk.webrtc.MagicPeerConnectionWrapper;
import org.apache.commons.lang3.StringEscapeUtils;
import org.greenrobot.eventbus.EventBus;
@ -104,8 +106,8 @@ public class CallActivity extends AppCompatActivity {
@BindView(R.id.pip_video_view)
SurfaceViewRenderer pipVideoView;
@BindView(R.id.fullscreen_video_view)
SurfaceViewRenderer fullScreenVideoView;
@BindView(R.id.videos_grid_view)
GridLayout videosGrid;
@Inject
NcApi ncApi;
@ -125,7 +127,9 @@ public class CallActivity extends AppCompatActivity {
VideoCapturer videoCapturer;
VideoRenderer localRenderer;
VideoRenderer remoteRenderer;
HashMap<String, VideoRenderer> videoRendererHashMap = new HashMap<>();
PeerConnection localPeer;
EglBase rootEglBase;
boolean leavingCall = false;
BooleanSupplier booleanSupplier = () -> leavingCall;
Disposable signalingDisposable;
@ -138,7 +142,7 @@ public class CallActivity extends AppCompatActivity {
private MediaStream localMediaStream;
private String credentials;
private List<PeerConnectionWrapper> peerConnectionWrapperList = new ArrayList<>();
private List<MagicPeerConnectionWrapper> magicPeerConnectionWrapperList = new ArrayList<>();
private static int getSystemUiVisibility() {
int flags = View.SYSTEM_UI_FLAG_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_FULLSCREEN;
@ -224,16 +228,11 @@ public class CallActivity extends AppCompatActivity {
public void initViews() {
pipVideoView.setMirror(true);
fullScreenVideoView.setMirror(false);
EglBase rootEglBase = EglBase.create();
rootEglBase = EglBase.create();
pipVideoView.init(rootEglBase.getEglBaseContext(), null);
pipVideoView.setZOrderMediaOverlay(true);
fullScreenVideoView.init(rootEglBase.getEglBaseContext(), null);
fullScreenVideoView.setZOrderMediaOverlay(true);
fullScreenVideoView.setEnableHardwareScaler(true);
pipVideoView.setEnableHardwareScaler(true);
pipVideoView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT);
fullScreenVideoView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL);
}
public void start() {
@ -435,7 +434,7 @@ public class CallActivity extends AppCompatActivity {
NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(),
NCSignalingMessage.class);
if (ncSignalingMessage.getRoomType().equals("video")) {
PeerConnectionWrapper peerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId
MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId
(ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession));
String type = null;
@ -450,8 +449,8 @@ public class CallActivity extends AppCompatActivity {
switch (type) {
case "offer":
case "answer":
peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper
magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
magicPeerConnectionWrapper.getPeerConnection().setRemoteDescription(magicPeerConnectionWrapper
.getMagicSdpObserver(), new SessionDescription(SessionDescription.Type.fromCanonicalForm(type),
ncSignalingMessage.getPayload().getSdp()));
break;
@ -459,11 +458,11 @@ public class CallActivity extends AppCompatActivity {
NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate();
IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(),
ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate());
peerConnectionWrapper.addCandidate(iceCandidate);
magicPeerConnectionWrapper.addCandidate(iceCandidate);
break;
case "endOfCandidates":
peerConnectionWrapper.drainIceCandidates();
peerConnectionWrapper.sendLocalCandidates();
magicPeerConnectionWrapper.drainIceCandidates();
magicPeerConnectionWrapper.sendLocalCandidates();
break;
default:
break;
@ -496,9 +495,9 @@ public class CallActivity extends AppCompatActivity {
}
}
for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
if (!peerConnectionWrapper.getSessionId().equals(callSession)) {
oldSesssions.add(peerConnectionWrapper.getSessionId());
for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
if (!magicPeerConnectionWrapper.getSessionId().equals(callSession)) {
oldSesssions.add(magicPeerConnectionWrapper.getSessionId());
}
}
@ -513,12 +512,12 @@ public class CallActivity extends AppCompatActivity {
return;
}
PeerConnectionWrapper peerConnectionWrapper;
MagicPeerConnectionWrapper magicPeerConnectionWrapper;
for (String sessionId : newSessions) {
if (getPeerConnectionWrapperForSessionId(sessionId) == null) {
if (sessionId.compareTo(callSession) < 0) {
PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId,
MagicPeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId,
false);
if (connectionWrapper.getPeerConnection() != null) {
connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(),
@ -532,33 +531,33 @@ public class CallActivity extends AppCompatActivity {
}
for (String sessionId : leftSessions) {
if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
if (peerConnectionWrapper.getPeerConnection() != null) {
peerConnectionWrapper.getPeerConnection().close();
if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
if (magicPeerConnectionWrapper.getPeerConnection() != null) {
magicPeerConnectionWrapper.getPeerConnection().close();
}
peerConnectionWrapperList.remove(peerConnectionWrapper);
magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper);
}
}
}
private PeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) {
PeerConnectionWrapper peerConnectionWrapper;
if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
return peerConnectionWrapper;
private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) {
MagicPeerConnectionWrapper magicPeerConnectionWrapper;
if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
return magicPeerConnectionWrapper;
} else {
peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
iceServers, sdpConstraints, sessionId);
peerConnectionWrapper.getPeerConnection().addStream(localMediaStream);
peerConnectionWrapperList.add(peerConnectionWrapper);
return peerConnectionWrapper;
magicPeerConnectionWrapper.getPeerConnection().addStream(localMediaStream);
magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper);
return magicPeerConnectionWrapper;
}
}
private PeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) {
for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
if (peerConnectionWrapper.getSessionId().equals(sessionId)) {
return peerConnectionWrapper;
private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) {
for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) {
return magicPeerConnectionWrapper;
}
}
return null;
@ -570,9 +569,9 @@ public class CallActivity extends AppCompatActivity {
dispose(null);
for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
if (peerConnectionWrapper.getPeerConnection() != null) {
peerConnectionWrapper.getPeerConnection().close();
for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
if (magicPeerConnectionWrapper.getPeerConnection() != null) {
magicPeerConnectionWrapper.getPeerConnection().close();
}
}
@ -581,7 +580,6 @@ public class CallActivity extends AppCompatActivity {
}
pipVideoView.release();
fullScreenVideoView.release();
String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken());
ncApi.leaveCall(credentials, ApiHelper.getUrlForCall(userEntity.getBaseUrl(), roomToken))
@ -610,7 +608,7 @@ public class CallActivity extends AppCompatActivity {
});
}
private void gotRemoteStream(MediaStream stream) {
private void gotRemoteStream(MediaStream stream, String session) {
//we have remote video stream. add to the renderer.
if (stream.videoTracks.size() < 2 && stream.audioTracks.size() < 2) {
if (stream.videoTracks.size() == 1) {
@ -621,8 +619,26 @@ public class CallActivity extends AppCompatActivity {
public void run() {
if (stream.videoTracks.size() == 1) {
try {
remoteRenderer = new VideoRenderer(fullScreenVideoView);
RelativeLayout relativeLayout = (RelativeLayout)
getLayoutInflater().inflate(R.layout.surface_renderer, videosGrid,
false);
relativeLayout.setTag(session);
SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id
.surface_view);
surfaceViewRenderer.setMirror(false);
surfaceViewRenderer.init(rootEglBase.getEglBaseContext(), null);
surfaceViewRenderer.setZOrderMediaOverlay(true);
surfaceViewRenderer.setEnableHardwareScaler(true);
surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL);
VideoRenderer remoteRenderer = new VideoRenderer(surfaceViewRenderer);
videoRendererHashMap.put(session, remoteRenderer);
videoTrack.addRenderer(remoteRenderer);
videosGrid.addView(relativeLayout);
GridLayout.LayoutParams param = new GridLayout.LayoutParams(GridLayout.spec(
GridLayout.UNDEFINED,GridLayout.FILL,1f),
GridLayout.spec(GridLayout.UNDEFINED,GridLayout.FILL,1f));
relativeLayout.setLayoutParams(param);
} catch (Exception e) {
e.printStackTrace();
}
@ -676,7 +692,7 @@ public class CallActivity extends AppCompatActivity {
@Subscribe(threadMode = ThreadMode.BACKGROUND)
public void onMessageEvent(MediaStreamEvent mediaStreamEvent) {
gotRemoteStream(mediaStreamEvent.getMediaStream());
gotRemoteStream(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession());
}
@Subscribe(threadMode = ThreadMode.BACKGROUND)

7
app/src/main/java/com/nextcloud/talk/api/models/json/signaling/DataChannelMessage.java
View File

@ -33,4 +33,11 @@ public class DataChannelMessage {
@JsonField(name = "payload")
String payload;
public DataChannelMessage(String type) {
this.type = type;
}
public DataChannelMessage() {
}
}

4
app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java
View File

@ -27,8 +27,10 @@ import lombok.Data;
@Data
public class MediaStreamEvent {
private final MediaStream mediaStream;
private final String session;
public MediaStreamEvent(MediaStream mediaStream) {
public MediaStreamEvent(MediaStream mediaStream, String session) {
this.mediaStream = mediaStream;
this.session = session;
}
}

3
app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java
View File

@ -20,8 +20,6 @@
package com.nextcloud.talk.webrtc;
import android.util.Log;
import org.webrtc.DataChannel;
import org.webrtc.IceCandidate;
import org.webrtc.MediaStream;
@ -35,7 +33,6 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer {
@Override
public void onSignalingChange(PeerConnection.SignalingState signalingState) {
Log.d("MARIO", signalingState.name());
}
@Override

91
app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java → app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java
View File

@ -22,6 +22,7 @@ package com.nextcloud.talk.webrtc;
import android.util.Log;
import com.bluelinelabs.logansquare.LoganSquare;
import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage;
import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate;
import com.nextcloud.talk.events.MediaStreamEvent;
@ -36,12 +37,13 @@ import org.webrtc.PeerConnection;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.SessionDescription;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
public class PeerConnectionWrapper {
private static String TAG = "PeerConnectionWrapper";
public class MagicPeerConnectionWrapper {
private static String TAG = "MagicPeerConnectionWrapper";
private static PeerConnection peerConnection;
List<IceCandidate> iceCandidates = new ArrayList<>();
List<PeerConnection.IceServer> iceServers;
@ -49,39 +51,21 @@ public class PeerConnectionWrapper {
private String sessionId;
private String nick;
private MediaConstraints mediaConstraints;
private DataChannel dataChannel;
private DataChannel magicDataChannel;
private MagicSdpObserver magicSdpObserver;
private MagicPeerConnectionObserver magicPeerConnectionObserver;
public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
List<PeerConnection.IceServer> iceServerList,
MediaConstraints mediaConstraints,
String sessionId) {
public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
List<PeerConnection.IceServer> iceServerList,
MediaConstraints mediaConstraints,
String sessionId) {
this.iceServers = iceServerList;
magicPeerConnectionObserver = new MagicPeerConnectionObserver() {
@Override
public void onIceConnectionReceivingChange(boolean b) {
}
@Override
public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
}
@Override
public void onAddStream(MediaStream mediaStream) {
EventBus.getDefault().post(new MediaStreamEvent(mediaStream));
}
@Override
public void onSignalingChange(PeerConnection.SignalingState signalingState) {
}
@Override
public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
EventBus.getDefault().post(new MediaStreamEvent(mediaStream, sessionId));
}
@Override
@ -90,12 +74,15 @@ public class PeerConnectionWrapper {
ncIceCandidate.setSdpMid(iceCandidate.sdpMid);
ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex);
ncIceCandidate.setCandidate(iceCandidate.sdp);
if (peerConnection.getRemoteDescription() == null) {
/*if (peerConnection.getRemoteDescription() == null) {
localCandidates.add(ncIceCandidate);
} else {
EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
"candidate", ncIceCandidate));
}
}*/
EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
"candidate", ncIceCandidate));
}
};
@ -103,6 +90,11 @@ public class PeerConnectionWrapper {
peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints,
magicPeerConnectionObserver);
DataChannel.Init init = new DataChannel.Init();
init.negotiated = false;
magicDataChannel = peerConnection.createDataChannel("status", init);
magicDataChannel.registerObserver(new MagicDataChannelObserver());
this.sessionId = sessionId;
this.mediaConstraints = mediaConstraints;
@ -120,6 +112,7 @@ public class PeerConnectionWrapper {
@Override
public void onCreateSuccess(SessionDescription sessionDescription) {
super.onCreateSuccess(sessionDescription);
peerConnection.setLocalDescription(magicSdpObserver, sessionDescription);
}
@ -181,8 +174,13 @@ public class PeerConnectionWrapper {
}
private void sendChannelData(DataChannelMessage dataChannelMessage) {
ByteBuffer buffer = ByteBuffer.wrap(dataChannelMessage.toString().getBytes());
dataChannel.send(new DataChannel.Buffer(buffer, false));
ByteBuffer buffer = null;
try {
buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes());
magicDataChannel.send(new DataChannel.Buffer(buffer, false));
} catch (IOException e) {
e.printStackTrace();
}
}
@ -191,7 +189,7 @@ public class PeerConnectionWrapper {
}
public static void setPeerConnection(PeerConnection peerConnection) {
PeerConnectionWrapper.peerConnection = peerConnection;
MagicPeerConnectionWrapper.peerConnection = peerConnection;
}
public String getSessionId() {
@ -209,4 +207,35 @@ public class PeerConnectionWrapper {
public void setNick(String nick) {
this.nick = nick;
}
private class MagicDataChannelObserver implements DataChannel.Observer {
@Override
public void onBufferedAmountChange(long l) {
}
@Override
public void onStateChange() {
if (magicDataChannel.state().equals(DataChannel.State.OPEN) &&
magicDataChannel.label().equals("status")) {
sendChannelData(new DataChannelMessage("videoOn"));
sendChannelData(new DataChannelMessage("audioOn"));
}
}
@Override
public void onMessage(DataChannel.Buffer buffer) {
if (buffer.binary) {
Log.d(TAG, "Received binary msg over " + TAG + " " + sessionId);
return;
}
ByteBuffer data = buffer.data;
final byte[] bytes = new byte[data.capacity()];
data.get(bytes);
String strData = new String(bytes);
Log.d(TAG, "Got msg: " + strData + " over " + TAG + " " + sessionId);
}
}
}

14
app/src/main/res/layout/activity_call.xml
View File

@ -26,6 +26,15 @@
android:fitsSystemWindows="true"
tools:context=".activities.CallActivity">
<GridLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:id="@+id/videos_grid_view"
android:columnCount="4"
android:rowCount="2">
</GridLayout>
<org.webrtc.SurfaceViewRenderer
android:id="@+id/pip_video_view"
android:layout_width="120dp"
@ -34,9 +43,4 @@
android:layout_alignParentEnd="true"
android:layout_margin="16dp"/>
<org.webrtc.SurfaceViewRenderer
android:id="@+id/fullscreen_video_view"
android:layout_width="match_parent"
android:layout_height="match_parent"/>
</RelativeLayout>

33
app/src/main/res/layout/surface_renderer.xml Normal file
View File

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Nextcloud Talk application
~
~ @author Mario Danic
~ Copyright (C) 2017 Mario Danic <mario@lovelyhq.com>
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="@+id/relative_layout"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical">
<org.webrtc.SurfaceViewRenderer
android:layout_width="match_parent"
android:layout_height="match_parent"
android:id="@+id/surface_view"/>
</RelativeLayout>