From 82e5d69000afe40f4d4604107e05898d9a1d0c4a Mon Sep 17 00:00:00 2001 From: Mario Danic Date: Thu, 21 Dec 2017 03:14:16 +0100 Subject: [PATCH] Fix #38 Signed-off-by: Mario Danic --- .../talk/activities/CallActivity.java | 252 ++++++++++++++---- .../talk/events/PeerConnectionEvent.java | 9 +- .../talk/webrtc/MagicAudioManager.java | 4 +- .../webrtc/MagicPeerConnectionWrapper.java | 51 ++-- app/src/main/res/layout/activity_call.xml | 49 +++- app/src/main/res/layout/surface_renderer.xml | 55 ++-- app/src/main/res/values/colors.xml | 3 +- 7 files changed, 326 insertions(+), 97 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index b3601a28b..61b026b6d 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -25,6 +25,8 @@ package com.nextcloud.talk.activities; import android.Manifest; +import android.animation.Animator; +import android.animation.AnimatorListenerAdapter; import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; @@ -42,6 +44,8 @@ import android.util.TypedValue; import android.view.View; import android.view.Window; import android.view.WindowManager; +import android.widget.ImageButton; +import android.widget.ImageView; import android.widget.LinearLayout; import android.widget.RelativeLayout; import android.widget.TextView; @@ -111,6 +115,7 @@ import javax.inject.Inject; import autodagger.AutoInjector; import butterknife.BindView; import butterknife.ButterKnife; +import butterknife.OnClick; import io.reactivex.Observer; import io.reactivex.android.schedulers.AndroidSchedulers; import io.reactivex.disposables.Disposable; @@ -136,6 +141,16 @@ public class CallActivity extends AppCompatActivity { RelativeLayout relativeLayout; @BindView(R.id.remote_renderers_layout) LinearLayout remoteRenderersLayout; + + @BindView(R.id.call_controls) + RelativeLayout callControls; + @BindView(R.id.call_control_microphone) + ImageButton microphoneControlButton; + @BindView(R.id.call_control_camera) + ImageButton cameraControlButton; + @BindView(R.id.call_control_switch_camera) + ImageButton cameraSwitchButton; + @Inject NcApi ncApi; @Inject @@ -157,6 +172,7 @@ public class CallActivity extends AppCompatActivity { Disposable signalingDisposable; Disposable pingDisposable; List iceServers; + private CameraEnumerator cameraEnumerator; private String roomToken; private UserEntity userEntity; private String callSession; @@ -165,6 +181,9 @@ public class CallActivity extends AppCompatActivity { private String credentials; private List magicPeerConnectionWrapperList = new ArrayList<>(); + private boolean videoOn = true; + private boolean audioOn = true; + private static int getSystemUiVisibility() { int flags = View.SYSTEM_UI_FLAG_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_FULLSCREEN; flags |= View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY; @@ -213,6 +232,12 @@ public class CallActivity extends AppCompatActivity { } localMediaStream.videoTracks.get(0).setEnabled(enable); + + if (enable) { + pipVideoView.setVisibility(View.VISIBLE); + } else { + pipVideoView.setVisibility(View.INVISIBLE); + } } else { message = "audioOff"; if (enable) { @@ -227,21 +252,55 @@ public class CallActivity extends AppCompatActivity { } } - private void switchCamera() { + @OnClick(R.id.call_control_microphone) + public void onMicrophoneClick() { + audioOn = !audioOn; + + if (audioOn) { + microphoneControlButton.setImageResource(R.drawable.ic_mic_white_24px); + } else { + microphoneControlButton.setImageResource(R.drawable.ic_mic_off_white_24px); + } + + toggleMedia(audioOn, false); + } + + @OnClick(R.id.call_control_hangup) + public void onHangupClick() { + hangup(false); + finish(); + } + + @OnClick(R.id.call_control_camera) + public void onCameraClick() { + videoOn = !videoOn; + + if (videoOn) { + cameraControlButton.setImageResource(R.drawable.ic_videocam_white_24px); + } else { + cameraControlButton.setImageResource(R.drawable.ic_videocam_off_white_24px); + } + + toggleMedia(videoOn, true); + } + + + @OnClick(R.id.call_control_switch_camera) + public void switchCamera() { CameraVideoCapturer cameraVideoCapturer = (CameraVideoCapturer) videoCapturer; cameraVideoCapturer.switchCamera(null); } - private VideoCapturer createVideoCapturer() { - CameraEnumerator cameraEnumerator; - + private void createCameraEnumerator() { if (Camera2Enumerator.isSupported(this)) { cameraEnumerator = new Camera2Enumerator(this); } else { cameraEnumerator = new Camera1Enumerator(false); } - videoCapturer = createCameraCapturer(cameraEnumerator); + } + private VideoCapturer createVideoCapturer() { + videoCapturer = createCameraCapturer(cameraEnumerator); return videoCapturer; } @@ -278,6 +337,12 @@ public class CallActivity extends AppCompatActivity { } public void initViews() { + createCameraEnumerator(); + + if (cameraEnumerator.getDeviceNames().length < 2) { + cameraSwitchButton.setVisibility(View.GONE); + } + // setting this to true because it's not shown by default pipVideoView.setMirror(true); rootEglBase = EglBase.create(); @@ -372,14 +437,24 @@ public class CallActivity extends AppCompatActivity { sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true")); sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); + animateCallControls(false, 5000); startPullingSignalingMessages(false); registerNetworkReceiver(); } + + @OnClick({R.id.full_screen_surface_view, R.id.remote_renderers_layout}) + public void showCallControls() { + if (callControls.getVisibility() != View.VISIBLE) { + animateCallControls(true, 0); + } + } + public void startPullingSignalingMessages(boolean restart) { if (restart) { dispose(null); + hangupNetworkCalls(); } leavingCall = false; @@ -744,58 +819,62 @@ public class CallActivity extends AppCompatActivity { pipVideoView.release(); if (!dueToNetworkChange) { - String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()); - ncApi.leaveCall(credentials, ApiHelper.getUrlForCall(userEntity.getBaseUrl(), roomToken)) - .subscribeOn(Schedulers.newThread()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe(new Observer() { - @Override - public void onSubscribe(Disposable d) { - - } - - @Override - public void onNext(GenericOverall genericOverall) { - ncApi.leaveRoom(credentials, ApiHelper.getUrlForRoom(userEntity.getBaseUrl(), roomToken)) - .subscribeOn(Schedulers.newThread()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe(new Observer() { - @Override - public void onSubscribe(Disposable d) { - - } - - @Override - public void onNext(GenericOverall genericOverall) { - - } - - @Override - public void onError(Throwable e) { - - } - - @Override - public void onComplete() { - - } - }); - - } - - @Override - public void onError(Throwable e) { - - } - - @Override - public void onComplete() { - - } - }); + hangupNetworkCalls(); } } + private void hangupNetworkCalls() { + String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()); + ncApi.leaveCall(credentials, ApiHelper.getUrlForCall(userEntity.getBaseUrl(), roomToken)) + .subscribeOn(Schedulers.newThread()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe(new Observer() { + @Override + public void onSubscribe(Disposable d) { + + } + + @Override + public void onNext(GenericOverall genericOverall) { + ncApi.leaveRoom(credentials, ApiHelper.getUrlForRoom(userEntity.getBaseUrl(), roomToken)) + .subscribeOn(Schedulers.newThread()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe(new Observer() { + @Override + public void onSubscribe(Disposable d) { + + } + + @Override + public void onNext(GenericOverall genericOverall) { + + } + + @Override + public void onError(Throwable e) { + + } + + @Override + public void onComplete() { + + } + }); + + } + + @Override + public void onError(Throwable e) { + + } + + @Override + public void onComplete() { + + } + }); + } + private void gotNick(String sessionId, String nick) { RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(sessionId); if (relativeLayout != null) { @@ -804,6 +883,24 @@ public class CallActivity extends AppCompatActivity { } } + private void gotAudioOrVideoChange(boolean video, String sessionId, boolean change) { + RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(sessionId); + if (relativeLayout != null) { + ImageView imageView; + if (video) { + imageView = relativeLayout.findViewById(R.id.remote_video_off); + } else { + imageView = relativeLayout.findViewById(R.id.remote_audio_off); + } + + if (change && imageView.getVisibility() != View.INVISIBLE) { + imageView.setVisibility(View.INVISIBLE); + } else if (!change && imageView.getVisibility() != View.VISIBLE) { + imageView.setVisibility(View.VISIBLE); + } + } + } + private void gotRemoteStream(MediaStream stream, String session) { if (fullScreenVideoView != null) { remoteRenderersLayout.setVisibility(View.VISIBLE); @@ -893,11 +990,20 @@ public class CallActivity extends AppCompatActivity { .PeerConnectionEventType.SENSOR_FAR) || peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent .PeerConnectionEventType.SENSOR_NEAR)) { - boolean enableVideo = peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent .PeerConnectionEventType.SENSOR_FAR); - toggleMedia(enableVideo, true); + } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent + .PeerConnectionEventType.NICK_CHANGE)) { + runOnUiThread(() -> gotNick(peerConnectionEvent.getSessionId(), peerConnectionEvent.getNick())); + } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent + .PeerConnectionEventType.VIDEO_CHANGE)) { + runOnUiThread(() -> gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId(), + peerConnectionEvent.getChangeValue())); + } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent + .PeerConnectionEventType.AUDIO_CHANGE)) { + runOnUiThread(() -> gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId(), + peerConnectionEvent.getChangeValue())); } } @@ -1046,4 +1152,38 @@ public class CallActivity extends AppCompatActivity { this.registerReceiver(broadcastReceiver, intentFilter); } + + private void animateCallControls(boolean show, long startDelay) { + float alpha; + long duration; + + if (show) { + alpha = 1.0f; + duration = 500; + } else { + alpha = 0.0f; + duration = 2500; + } + + callControls.animate() + .translationY(0) + .alpha(alpha) + .setDuration(duration) + .setStartDelay(startDelay) + .setListener(new AnimatorListenerAdapter() { + @Override + public void onAnimationEnd(Animator animation) { + super.onAnimationEnd(animation); + if (callControls != null) { + if (!show) { + callControls.setVisibility(View.INVISIBLE); + } else { + callControls.setVisibility(View.VISIBLE); + animateCallControls(false, 10000); + } + } + } + }); + + } } diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java index e3749f3aa..d5dcf7cdf 100644 --- a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java @@ -28,13 +28,18 @@ import lombok.Data; public class PeerConnectionEvent { private final PeerConnectionEventType peerConnectionEventType; private final String sessionId; + private final String nick; + private final Boolean changeValue; - public PeerConnectionEvent(PeerConnectionEventType peerConnectionEventType, @Nullable String sessionId) { + public PeerConnectionEvent(PeerConnectionEventType peerConnectionEventType, @Nullable String sessionId, + @Nullable String nick, Boolean changeValue) { this.peerConnectionEventType = peerConnectionEventType; + this.nick = nick; + this.changeValue = changeValue; this.sessionId = sessionId; } public enum PeerConnectionEventType { - CLOSE_PEER, SENSOR_FAR, SENSOR_NEAR + CLOSE_PEER, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE } } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java index e4d069dcc..9c400797e 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java @@ -143,10 +143,10 @@ public class MagicAudioManager { if (proximitySensor.sensorReportsNearState()) { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .SENSOR_NEAR, null)); + .SENSOR_NEAR, null, null, null)); } else { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .SENSOR_FAR, null)); + .SENSOR_FAR, null, null, null)); } if (!useSpeakerphone.equals(SPEAKERPHONE_AUTO)) { diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java index 65811a9da..0503df7ab 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java @@ -58,8 +58,8 @@ public class MagicPeerConnectionWrapper { private DataChannel magicDataChannel; private MagicSdpObserver magicSdpObserver; - private boolean audioOn; - private boolean videoOn; + private boolean remoteVideoOn; + private boolean remoteAudioOn; private boolean hasInitiated; @@ -167,14 +167,41 @@ public class MagicPeerConnectionWrapper { Log.d(TAG, "Received binary msg over " + TAG + " " + sessionId); return; } + ByteBuffer data = buffer.data; final byte[] bytes = new byte[data.capacity()]; data.get(bytes); String strData = new String(bytes); Log.d(TAG, "Got msg: " + strData + " over " + TAG + " " + sessionId); - // We use media stream to determine if audio or video is on rather than data - // channel messages + try { + DataChannelMessage dataChannelMessage = LoganSquare.parse(strData, DataChannelMessage.class); + + if ("nickChanged".equals(dataChannelMessage.getType())) { + nick = dataChannelMessage.getPayload(); + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType + .NICK_CHANGE, sessionId, nick, null)); + } else if ("audioOn".equals(dataChannelMessage.getType())) { + remoteAudioOn = true; + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType + .AUDIO_CHANGE, sessionId, null, remoteAudioOn)); + } else if ("audioOff".equals(dataChannelMessage.getType())) { + remoteAudioOn = false; + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType + .AUDIO_CHANGE, sessionId, null, remoteAudioOn)); + } else if ("videoOn".equals(dataChannelMessage.getType())) { + remoteVideoOn = true; + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType + .VIDEO_CHANGE, sessionId, null, remoteVideoOn)); + } else if ("videoOff".equals(dataChannelMessage.getType())) { + remoteVideoOn = false; + EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType + .VIDEO_CHANGE, sessionId, null, remoteVideoOn)); + } + + } catch (IOException e) { + Log.d(TAG, "Failed to parse data channel message"); + } } } @@ -185,7 +212,7 @@ public class MagicPeerConnectionWrapper { public void onSignalingChange(PeerConnection.SignalingState signalingState) { if (signalingState.equals(PeerConnection.SignalingState.CLOSED)) { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .CLOSE_PEER, sessionId)); + .CLOSE_PEER, sessionId, null, null)); } } @@ -196,7 +223,7 @@ public class MagicPeerConnectionWrapper { sendChannelData(new DataChannelMessage("audioOn")); } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.FAILED)) { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .CLOSE_PEER, sessionId)); + .CLOSE_PEER, sessionId, null, null)); } } @@ -227,20 +254,12 @@ public class MagicPeerConnectionWrapper { @Override public void onAddStream(MediaStream mediaStream) { - videoOn = mediaStream.videoTracks != null && mediaStream.videoTracks.size() == 1; - audioOn = mediaStream.audioTracks != null && mediaStream.audioTracks.size() == 1; - if (!sessionId.equals(localSession)) { - EventBus.getDefault().post(new MediaStreamEvent(mediaStream, sessionId)); - } + EventBus.getDefault().post(new MediaStreamEvent(mediaStream, sessionId)); } @Override public void onRemoveStream(MediaStream mediaStream) { - videoOn = mediaStream.videoTracks != null && mediaStream.videoTracks.size() == 1; - audioOn = mediaStream.audioTracks != null && mediaStream.audioTracks.size() == 1; - if (!sessionId.equals(localSession)) { - EventBus.getDefault().post(new MediaStreamEvent(null, sessionId)); - } + EventBus.getDefault().post(new MediaStreamEvent(null, sessionId)); } @Override diff --git a/app/src/main/res/layout/activity_call.xml b/app/src/main/res/layout/activity_call.xml index 9a96dde9c..30fdc426c 100644 --- a/app/src/main/res/layout/activity_call.xml +++ b/app/src/main/res/layout/activity_call.xml @@ -28,9 +28,9 @@ tools:context=".activities.CallActivity"> + android:layout_height="match_parent"/> + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/surface_renderer.xml b/app/src/main/res/layout/surface_renderer.xml index b5e0030bf..63c707d53 100644 --- a/app/src/main/res/layout/surface_renderer.xml +++ b/app/src/main/res/layout/surface_renderer.xml @@ -20,24 +20,45 @@ --> + android:id="@+id/relative_layout" + android:layout_width="match_parent" + android:layout_height="match_parent" + android:layout_weight="1" + android:orientation="vertical"> - + - + + + + + \ No newline at end of file diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml index d4424af7d..0727eb745 100644 --- a/app/src/main/res/values/colors.xml +++ b/app/src/main/res/values/colors.xml @@ -7,7 +7,6 @@ #D32F2F @color/per70white #FFFFFF - #7fC0E3 - + #7FC0E3