diff --git a/changelog.d/5911.feature b/changelog.d/5911.feature new file mode 100644 index 0000000000..368a3b4056 --- /dev/null +++ b/changelog.d/5911.feature @@ -0,0 +1 @@ +Screen sharing over WebRTC diff --git a/vector/src/main/java/im/vector/app/features/VectorFeatures.kt b/vector/src/main/java/im/vector/app/features/VectorFeatures.kt index 9d54475e8c..42693a53f9 100644 --- a/vector/src/main/java/im/vector/app/features/VectorFeatures.kt +++ b/vector/src/main/java/im/vector/app/features/VectorFeatures.kt @@ -44,5 +44,5 @@ class DefaultVectorFeatures : VectorFeatures { override fun isOnboardingPersonalizeEnabled() = false override fun isOnboardingCombinedRegisterEnabled() = false override fun isLiveLocationEnabled(): Boolean = false - override fun isScreenSharingEnabled(): Boolean = false + override fun isScreenSharingEnabled(): Boolean = true } diff --git a/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt index b3fc36e5bc..f0158fc4d6 100644 --- a/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt @@ -89,6 +89,8 @@ class CallControlsView @JvmOverloads constructor( views.videoToggleIcon.setImageResource(R.drawable.ic_video_off) views.videoToggleIcon.contentDescription = resources.getString(R.string.a11y_start_camera) } + views.videoToggleIcon.isEnabled = !state.isSharingScreen + views.videoToggleIcon.alpha = if (state.isSharingScreen) 0.5f else 1f when (callState) { is CallState.LocalRinging -> { diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index ea9adcde85..a904658e9c 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -24,6 +24,7 @@ import android.content.Intent import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP import android.content.res.Configuration import android.graphics.Color +import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.Build import android.os.Bundle @@ -32,6 +33,7 @@ import android.util.Rational import android.view.MenuItem import android.view.View import android.view.WindowManager +import androidx.activity.result.ActivityResult import androidx.annotation.StringRes import androidx.core.content.ContextCompat import androidx.core.content.getSystemService @@ -76,6 +78,7 @@ import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.session.room.model.call.EndCallReason import org.webrtc.EglBase import org.webrtc.RendererCommon +import org.webrtc.ScreenCapturerAndroid import timber.log.Timber import javax.inject.Inject @@ -161,6 +164,9 @@ class VectorCallActivity : VectorBaseActivity(), CallContro } } } + + // Bind to service in case of user killed the app while there is an ongoing call + bindToScreenCaptureService() } override fun onNewIntent(intent: Intent?) { @@ -636,18 +642,40 @@ class VectorCallActivity : VectorBaseActivity(), CallContro private val screenSharingPermissionActivityResultLauncher = registerStartForActivityResult { activityResult -> if (activityResult.resultCode == Activity.RESULT_OK) { - callViewModel.handle(VectorCallViewActions.StartScreenSharing) - // We need to start a foreground service with a sticky notification during screen sharing if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - ContextCompat.startForegroundService( - this, - Intent(this, ScreenCaptureService::class.java) - ) - screenCaptureServiceConnection.bind() + // We need to start a foreground service with a sticky notification during screen sharing + startScreenSharingService(activityResult) + } else { + startScreenSharing(activityResult) } } } + private fun startScreenSharing(activityResult: ActivityResult) { + val videoCapturer = ScreenCapturerAndroid(activityResult.data, object : MediaProjection.Callback() { + override fun onStop() { + Timber.i("User revoked the screen capturing permission") + } + }) + callViewModel.handle(VectorCallViewActions.StartScreenSharing(videoCapturer)) + } + + private fun startScreenSharingService(activityResult: ActivityResult) { + ContextCompat.startForegroundService( + this, + Intent(this, ScreenCaptureService::class.java) + ) + bindToScreenCaptureService(activityResult) + } + + private fun bindToScreenCaptureService(activityResult: ActivityResult? = null) { + screenCaptureServiceConnection.bind(object : ScreenCaptureServiceConnection.Callback { + override fun onServiceConnected() { + activityResult?.let { startScreenSharing(it) } + } + }) + } + private fun handleShowScreenSharingPermissionDialog() { getSystemService()?.let { navigator.openScreenSharingPermissionDialog(it.createScreenCaptureIntent(), screenSharingPermissionActivityResultLauncher) diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt index c84f733b9a..cec118f296 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt @@ -19,6 +19,7 @@ package im.vector.app.features.call import im.vector.app.core.platform.VectorViewModelAction import im.vector.app.features.call.audio.CallAudioManager import im.vector.app.features.call.transfer.CallTransferResult +import org.webrtc.VideoCapturer sealed class VectorCallViewActions : VectorViewModelAction { object EndCall : VectorCallViewActions() @@ -41,5 +42,5 @@ sealed class VectorCallViewActions : VectorViewModelAction { data class CallTransferSelectionResult(val callTransferResult: CallTransferResult) : VectorCallViewActions() object TransferCall : VectorCallViewActions() object ToggleScreenSharing : VectorCallViewActions() - object StartScreenSharing : VectorCallViewActions() + data class StartScreenSharing(val videoCapturer: VideoCapturer) : VectorCallViewActions() } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 55a0219bfe..e2cedbe1b0 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -145,9 +145,10 @@ class VectorCallViewModel @AssistedInject constructor( override fun onCallEnded(callId: String) { withState { state -> if (state.otherKnownCallInfo?.callId == callId) { - setState { copy(otherKnownCallInfo = null) } + setState { copy(otherKnownCallInfo = null, isSharingScreen = false) } } } + _viewEvents.post(VectorCallViewEvents.StopScreenSharingService) } override fun onCurrentCallChange(call: WebRtcCall?) { @@ -156,9 +157,10 @@ class VectorCallViewModel @AssistedInject constructor( } } - override fun onAudioDevicesChange() { - val currentSoundDevice = callManager.audioManager.selectedDevice ?: return - if (currentSoundDevice == CallAudioManager.Device.Phone) { + override fun onAudioDevicesChange() = withState { state -> + val currentSoundDevice = callManager.audioManager.selectedDevice ?: return@withState + val webRtcCall = callManager.getCallById(state.callId) + if (webRtcCall != null && shouldActivateProximitySensor(webRtcCall)) { proximityManager.start() } else { proximityManager.stop() @@ -205,7 +207,7 @@ class VectorCallViewModel @AssistedInject constructor( callManager.addListener(callManagerListener) webRtcCall.addListener(callListener) val currentSoundDevice = callManager.audioManager.selectedDevice - if (currentSoundDevice == CallAudioManager.Device.Phone) { + if (shouldActivateProximitySensor(webRtcCall)) { proximityManager.start() } setState { @@ -224,13 +226,18 @@ class VectorCallViewModel @AssistedInject constructor( formattedDuration = webRtcCall.formattedDuration(), isHD = webRtcCall.mxCall.isVideoCall && webRtcCall.currentCaptureFormat() is CaptureFormat.HD, canOpponentBeTransferred = webRtcCall.mxCall.capabilities.supportCallTransfer(), - transferee = computeTransfereeState(webRtcCall.mxCall) + transferee = computeTransfereeState(webRtcCall.mxCall), + isSharingScreen = webRtcCall.isSharingScreen() ) } updateOtherKnownCall(webRtcCall) } } + private fun shouldActivateProximitySensor(webRtcCall: WebRtcCall): Boolean { + return callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone && !webRtcCall.isSharingScreen() + } + private fun WebRtcCall.extractCallInfo(): VectorCallViewState.CallInfo { val assertedIdentity = this.remoteAssertedIdentity val matrixItem = if (assertedIdentity != null) { @@ -349,7 +356,8 @@ class VectorCallViewModel @AssistedInject constructor( handleToggleScreenSharing(state.isSharingScreen) } is VectorCallViewActions.StartScreenSharing -> { - call?.startSharingScreen() + call?.startSharingScreen(action.videoCapturer) + proximityManager.stop() setState { copy(isSharingScreen = true) } @@ -366,6 +374,9 @@ class VectorCallViewModel @AssistedInject constructor( _viewEvents.post( VectorCallViewEvents.StopScreenSharingService ) + if (callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone) { + proximityManager.start() + } } else { _viewEvents.post( VectorCallViewEvents.ShowScreenSharingPermissionDialog diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt index 922e9676a8..aa7c7f450a 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt @@ -27,11 +27,20 @@ class ScreenCaptureServiceConnection @Inject constructor( private val context: Context ) : ServiceConnection { + interface Callback { + fun onServiceConnected() + } + private var isBound = false private var screenCaptureService: ScreenCaptureService? = null + private var callback: Callback? = null - fun bind() { - if (!isBound) { + fun bind(callback: Callback) { + this.callback = callback + + if (isBound) { + callback.onServiceConnected() + } else { Intent(context, ScreenCaptureService::class.java).also { intent -> context.bindService(intent, this, 0) } @@ -45,10 +54,12 @@ class ScreenCaptureServiceConnection @Inject constructor( override fun onServiceConnected(className: ComponentName, binder: IBinder) { screenCaptureService = (binder as ScreenCaptureService.LocalBinder).getService() isBound = true + callback?.onServiceConnected() } override fun onServiceDisconnected(className: ComponentName) { isBound = false screenCaptureService = null + callback = null } } diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index f0db3e199f..5a100edcf2 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -80,10 +80,12 @@ import org.webrtc.MediaConstraints import org.webrtc.MediaStream import org.webrtc.PeerConnection import org.webrtc.PeerConnectionFactory +import org.webrtc.RtpSender import org.webrtc.RtpTransceiver import org.webrtc.SessionDescription import org.webrtc.SurfaceTextureHelper import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoCapturer import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber @@ -95,6 +97,7 @@ import kotlin.coroutines.CoroutineContext private const val STREAM_ID = "userMedia" private const val AUDIO_TRACK_ID = "${STREAM_ID}a0" private const val VIDEO_TRACK_ID = "${STREAM_ID}v0" +private const val SCREEN_TRACK_ID = "${STREAM_ID}s0" private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints() private const val INVITE_TIMEOUT_IN_MS = 60_000L @@ -153,13 +156,16 @@ class WebRtcCall( private var makingOffer: Boolean = false private var ignoreOffer: Boolean = false - private var videoCapturer: CameraVideoCapturer? = null + private var videoCapturer: VideoCapturer? = null private val availableCamera = ArrayList() private var cameraInUse: CameraProxy? = null private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null + private var videoSender: RtpSender? = null + private var screenSender: RtpSender? = null + private val timer = CountUpTimer(1000L).apply { tickListener = object : CountUpTimer.TickListener { override fun onTick(milliseconds: Long) { @@ -617,7 +623,7 @@ class WebRtcCall( val videoTrack = peerConnectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource) Timber.tag(loggerTag.value).v("Add video track $VIDEO_TRACK_ID to call ${mxCall.callId}") videoTrack.setEnabled(true) - peerConnection?.addTrack(videoTrack, listOf(STREAM_ID)) + videoSender = peerConnection?.addTrack(videoTrack, listOf(STREAM_ID)) localVideoSource = videoSource localVideoTrack = videoTrack } @@ -718,7 +724,7 @@ class WebRtcCall( Timber.tag(loggerTag.value).v("switchCamera") if (mxCall.state is CallState.Connected && mxCall.isVideoCall) { val oppositeCamera = getOppositeCameraIfAny() ?: return@launch - videoCapturer?.switchCamera( + (videoCapturer as? CameraVideoCapturer)?.switchCamera( object : CameraVideoCapturer.CameraSwitchHandler { // Invoked on success. |isFrontCamera| is true if the new camera is front facing. override fun onCameraSwitchDone(isFrontCamera: Boolean) { @@ -766,12 +772,60 @@ class WebRtcCall( return currentCaptureFormat } - fun startSharingScreen() { - // TODO. Will be handled within the next PR. + fun startSharingScreen(videoCapturer: VideoCapturer) { + val factory = peerConnectionFactoryProvider.get() ?: return + + this.videoCapturer = videoCapturer + + val localMediaStream = factory.createLocalMediaStream(STREAM_ID) + val videoSource = factory.createVideoSource(videoCapturer.isScreencast) + + startCapturingScreen(videoCapturer, videoSource) + + removeLocalSurfaceRenderers() + + showScreenLocally(factory, videoSource, localMediaStream) + + videoSender?.let { removeStream(it) } + + screenSender = peerConnection?.addTrack(localVideoTrack, listOf(STREAM_ID)) } fun stopSharingScreen() { - // TODO. Will be handled within the next PR. + localVideoTrack?.setEnabled(false) + screenSender?.let { removeStream(it) } + if (mxCall.isVideoCall) { + peerConnectionFactoryProvider.get()?.let { configureVideoTrack(it) } + } + updateMuteStatus() + sessionScope?.launch(dispatcher) { attachViewRenderersInternal() } + } + + private fun removeStream(sender: RtpSender) { + peerConnection?.removeTrack(sender) + } + + private fun showScreenLocally(factory: PeerConnectionFactory, videoSource: VideoSource?, localMediaStream: MediaStream?) { + localVideoTrack = factory.createVideoTrack(SCREEN_TRACK_ID, videoSource).apply { setEnabled(true) } + localMediaStream?.addTrack(localVideoTrack) + localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.addSink(it) } } + } + + private fun removeLocalSurfaceRenderers() { + localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.removeSink(it) } } + } + + private fun startCapturingScreen(videoCapturer: VideoCapturer, videoSource: VideoSource) { + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + videoCapturer.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps) + } + + /** + * Returns true if the user is sharing the screen, false otherwise. + */ + fun isSharingScreen(): Boolean { + return localVideoTrack?.enabled().orFalse() && localVideoTrack?.id() == SCREEN_TRACK_ID } private suspend fun release() {