Merge pull request #5911 from vector-im/feature/ons/voip_screen_sharing

Screen sharing over WebRTC

changelog.d/5911.feature

@@ -0,0 +1 @@
+Screen sharing over WebRTC

vector/src/main/java/im/vector/app/features/VectorFeatures.kt

@@ -44,5 +44,5 @@ class DefaultVectorFeatures : VectorFeatures {
     override fun isOnboardingPersonalizeEnabled() = false
     override fun isOnboardingCombinedRegisterEnabled() = false
     override fun isLiveLocationEnabled(): Boolean = false
-    override fun isScreenSharingEnabled(): Boolean = false
+    override fun isScreenSharingEnabled(): Boolean = true
 }

vector/src/main/java/im/vector/app/features/call/CallControlsView.kt

@@ -89,6 +89,8 @@ class CallControlsView @JvmOverloads constructor(
             views.videoToggleIcon.setImageResource(R.drawable.ic_video_off)
             views.videoToggleIcon.contentDescription = resources.getString(R.string.a11y_start_camera)
         }
+        views.videoToggleIcon.isEnabled = !state.isSharingScreen
+        views.videoToggleIcon.alpha = if (state.isSharingScreen) 0.5f else 1f
 
         when (callState) {
             is CallState.LocalRinging -> {

vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt

@@ -24,6 +24,7 @@ import android.content.Intent
 import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP
 import android.content.res.Configuration
 import android.graphics.Color
+import android.media.projection.MediaProjection
 import android.media.projection.MediaProjectionManager
 import android.os.Build
 import android.os.Bundle
@@ -32,6 +33,7 @@ import android.util.Rational
 import android.view.MenuItem
 import android.view.View
 import android.view.WindowManager
+import androidx.activity.result.ActivityResult
 import androidx.annotation.StringRes
 import androidx.core.content.ContextCompat
 import androidx.core.content.getSystemService
@@ -76,6 +78,7 @@ import org.matrix.android.sdk.api.session.call.TurnServerResponse
 import org.matrix.android.sdk.api.session.room.model.call.EndCallReason
 import org.webrtc.EglBase
 import org.webrtc.RendererCommon
+import org.webrtc.ScreenCapturerAndroid
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -161,6 +164,9 @@ class VectorCallActivity : VectorBaseActivity<ActivityCallBinding>(), CallContro
                 }
             }
         }
+
+        // Bind to service in case of user killed the app while there is an ongoing call
+        bindToScreenCaptureService()
     }
 
     override fun onNewIntent(intent: Intent?) {
@@ -636,16 +642,38 @@ class VectorCallActivity : VectorBaseActivity<ActivityCallBinding>(), CallContro
 
     private val screenSharingPermissionActivityResultLauncher = registerStartForActivityResult { activityResult ->
         if (activityResult.resultCode == Activity.RESULT_OK) {
-            callViewModel.handle(VectorCallViewActions.StartScreenSharing)
-            // We need to start a foreground service with a sticky notification during screen sharing
             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                // We need to start a foreground service with a sticky notification during screen sharing
+                startScreenSharingService(activityResult)
+            } else {
+                startScreenSharing(activityResult)
+            }
+        }
+    }
+
+    private fun startScreenSharing(activityResult: ActivityResult) {
+        val videoCapturer = ScreenCapturerAndroid(activityResult.data, object : MediaProjection.Callback() {
+            override fun onStop() {
+                Timber.i("User revoked the screen capturing permission")
+            }
+        })
+        callViewModel.handle(VectorCallViewActions.StartScreenSharing(videoCapturer))
+    }
+
+    private fun startScreenSharingService(activityResult: ActivityResult) {
         ContextCompat.startForegroundService(
                 this,
                 Intent(this, ScreenCaptureService::class.java)
         )
-                screenCaptureServiceConnection.bind()
+        bindToScreenCaptureService(activityResult)
     }
+
+    private fun bindToScreenCaptureService(activityResult: ActivityResult? = null) {
+        screenCaptureServiceConnection.bind(object : ScreenCaptureServiceConnection.Callback {
+            override fun onServiceConnected() {
+                activityResult?.let { startScreenSharing(it) }
             }
+        })
     }
 
     private fun handleShowScreenSharingPermissionDialog() {

vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt

@@ -19,6 +19,7 @@ package im.vector.app.features.call
 import im.vector.app.core.platform.VectorViewModelAction
 import im.vector.app.features.call.audio.CallAudioManager
 import im.vector.app.features.call.transfer.CallTransferResult
+import org.webrtc.VideoCapturer
 
 sealed class VectorCallViewActions : VectorViewModelAction {
     object EndCall : VectorCallViewActions()
@@ -41,5 +42,5 @@ sealed class VectorCallViewActions : VectorViewModelAction {
     data class CallTransferSelectionResult(val callTransferResult: CallTransferResult) : VectorCallViewActions()
     object TransferCall : VectorCallViewActions()
     object ToggleScreenSharing : VectorCallViewActions()
-    object StartScreenSharing : VectorCallViewActions()
+    data class StartScreenSharing(val videoCapturer: VideoCapturer) : VectorCallViewActions()
 }

vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt

@@ -145,9 +145,10 @@ class VectorCallViewModel @AssistedInject constructor(
         override fun onCallEnded(callId: String) {
             withState { state ->
                 if (state.otherKnownCallInfo?.callId == callId) {
-                    setState { copy(otherKnownCallInfo = null) }
+                    setState { copy(otherKnownCallInfo = null, isSharingScreen = false) }
                 }
             }
+            _viewEvents.post(VectorCallViewEvents.StopScreenSharingService)
         }
 
         override fun onCurrentCallChange(call: WebRtcCall?) {
@@ -156,9 +157,10 @@ class VectorCallViewModel @AssistedInject constructor(
             }
         }
 
-        override fun onAudioDevicesChange() {
+        override fun onAudioDevicesChange() = withState { state ->
-            val currentSoundDevice = callManager.audioManager.selectedDevice ?: return
+            val currentSoundDevice = callManager.audioManager.selectedDevice ?: return@withState
-            if (currentSoundDevice == CallAudioManager.Device.Phone) {
+            val webRtcCall = callManager.getCallById(state.callId)
+            if (webRtcCall != null && shouldActivateProximitySensor(webRtcCall)) {
                 proximityManager.start()
             } else {
                 proximityManager.stop()
@@ -205,7 +207,7 @@ class VectorCallViewModel @AssistedInject constructor(
             callManager.addListener(callManagerListener)
             webRtcCall.addListener(callListener)
             val currentSoundDevice = callManager.audioManager.selectedDevice
-            if (currentSoundDevice == CallAudioManager.Device.Phone) {
+            if (shouldActivateProximitySensor(webRtcCall)) {
                 proximityManager.start()
             }
             setState {
@@ -224,13 +226,18 @@ class VectorCallViewModel @AssistedInject constructor(
                         formattedDuration = webRtcCall.formattedDuration(),
                         isHD = webRtcCall.mxCall.isVideoCall && webRtcCall.currentCaptureFormat() is CaptureFormat.HD,
                         canOpponentBeTransferred = webRtcCall.mxCall.capabilities.supportCallTransfer(),
-                        transferee = computeTransfereeState(webRtcCall.mxCall)
+                        transferee = computeTransfereeState(webRtcCall.mxCall),
+                        isSharingScreen = webRtcCall.isSharingScreen()
                 )
             }
             updateOtherKnownCall(webRtcCall)
         }
     }
 
+    private fun shouldActivateProximitySensor(webRtcCall: WebRtcCall): Boolean {
+        return callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone && !webRtcCall.isSharingScreen()
+    }
+
     private fun WebRtcCall.extractCallInfo(): VectorCallViewState.CallInfo {
         val assertedIdentity = this.remoteAssertedIdentity
         val matrixItem = if (assertedIdentity != null) {
@@ -349,7 +356,8 @@ class VectorCallViewModel @AssistedInject constructor(
                 handleToggleScreenSharing(state.isSharingScreen)
             }
             is VectorCallViewActions.StartScreenSharing          -> {
-                call?.startSharingScreen()
+                call?.startSharingScreen(action.videoCapturer)
+                proximityManager.stop()
                 setState {
                     copy(isSharingScreen = true)
                 }
@@ -366,6 +374,9 @@ class VectorCallViewModel @AssistedInject constructor(
             _viewEvents.post(
                     VectorCallViewEvents.StopScreenSharingService
             )
+            if (callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone) {
+                proximityManager.start()
+            }
         } else {
             _viewEvents.post(
                     VectorCallViewEvents.ShowScreenSharingPermissionDialog

vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt

@@ -27,11 +27,20 @@ class ScreenCaptureServiceConnection @Inject constructor(
         private val context: Context
 ) : ServiceConnection {
 
+    interface Callback {
+        fun onServiceConnected()
+    }
+
     private var isBound = false
     private var screenCaptureService: ScreenCaptureService? = null
+    private var callback: Callback? = null
 
-    fun bind() {
+    fun bind(callback: Callback) {
-        if (!isBound) {
+        this.callback = callback
+
+        if (isBound) {
+            callback.onServiceConnected()
+        } else {
             Intent(context, ScreenCaptureService::class.java).also { intent ->
                 context.bindService(intent, this, 0)
             }
@@ -45,10 +54,12 @@ class ScreenCaptureServiceConnection @Inject constructor(
     override fun onServiceConnected(className: ComponentName, binder: IBinder) {
         screenCaptureService = (binder as ScreenCaptureService.LocalBinder).getService()
         isBound = true
+        callback?.onServiceConnected()
     }
 
     override fun onServiceDisconnected(className: ComponentName) {
         isBound = false
         screenCaptureService = null
+        callback = null
     }
 }

vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt

@@ -80,10 +80,12 @@ import org.webrtc.MediaConstraints
 import org.webrtc.MediaStream
 import org.webrtc.PeerConnection
 import org.webrtc.PeerConnectionFactory
+import org.webrtc.RtpSender
 import org.webrtc.RtpTransceiver
 import org.webrtc.SessionDescription
 import org.webrtc.SurfaceTextureHelper
 import org.webrtc.SurfaceViewRenderer
+import org.webrtc.VideoCapturer
 import org.webrtc.VideoSource
 import org.webrtc.VideoTrack
 import timber.log.Timber
@@ -95,6 +97,7 @@ import kotlin.coroutines.CoroutineContext
 private const val STREAM_ID = "userMedia"
 private const val AUDIO_TRACK_ID = "${STREAM_ID}a0"
 private const val VIDEO_TRACK_ID = "${STREAM_ID}v0"
+private const val SCREEN_TRACK_ID = "${STREAM_ID}s0"
 private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints()
 private const val INVITE_TIMEOUT_IN_MS = 60_000L
 
@@ -153,13 +156,16 @@ class WebRtcCall(
     private var makingOffer: Boolean = false
     private var ignoreOffer: Boolean = false
 
-    private var videoCapturer: CameraVideoCapturer? = null
+    private var videoCapturer: VideoCapturer? = null
 
     private val availableCamera = ArrayList<CameraProxy>()
     private var cameraInUse: CameraProxy? = null
     private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD
     private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null
 
+    private var videoSender: RtpSender? = null
+    private var screenSender: RtpSender? = null
+
     private val timer = CountUpTimer(1000L).apply {
         tickListener = object : CountUpTimer.TickListener {
             override fun onTick(milliseconds: Long) {
@@ -617,7 +623,7 @@ class WebRtcCall(
             val videoTrack = peerConnectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource)
             Timber.tag(loggerTag.value).v("Add video track $VIDEO_TRACK_ID to call ${mxCall.callId}")
             videoTrack.setEnabled(true)
-            peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
+            videoSender = peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
             localVideoSource = videoSource
             localVideoTrack = videoTrack
         }
@@ -718,7 +724,7 @@ class WebRtcCall(
             Timber.tag(loggerTag.value).v("switchCamera")
             if (mxCall.state is CallState.Connected && mxCall.isVideoCall) {
                 val oppositeCamera = getOppositeCameraIfAny() ?: return@launch
-                videoCapturer?.switchCamera(
+                (videoCapturer as? CameraVideoCapturer)?.switchCamera(
                         object : CameraVideoCapturer.CameraSwitchHandler {
                             // Invoked on success. |isFrontCamera| is true if the new camera is front facing.
                             override fun onCameraSwitchDone(isFrontCamera: Boolean) {
@@ -766,12 +772,60 @@ class WebRtcCall(
         return currentCaptureFormat
     }
 
-    fun startSharingScreen() {
+    fun startSharingScreen(videoCapturer: VideoCapturer) {
-        // TODO. Will be handled within the next PR.
+        val factory = peerConnectionFactoryProvider.get() ?: return
+
+        this.videoCapturer = videoCapturer
+
+        val localMediaStream = factory.createLocalMediaStream(STREAM_ID)
+        val videoSource = factory.createVideoSource(videoCapturer.isScreencast)
+
+        startCapturingScreen(videoCapturer, videoSource)
+
+        removeLocalSurfaceRenderers()
+
+        showScreenLocally(factory, videoSource, localMediaStream)
+
+        videoSender?.let { removeStream(it) }
+
+        screenSender = peerConnection?.addTrack(localVideoTrack, listOf(STREAM_ID))
     }
 
     fun stopSharingScreen() {
-        // TODO. Will be handled within the next PR.
+        localVideoTrack?.setEnabled(false)
+        screenSender?.let { removeStream(it) }
+        if (mxCall.isVideoCall) {
+            peerConnectionFactoryProvider.get()?.let { configureVideoTrack(it) }
+        }
+        updateMuteStatus()
+        sessionScope?.launch(dispatcher) { attachViewRenderersInternal() }
+    }
+
+    private fun removeStream(sender: RtpSender) {
+        peerConnection?.removeTrack(sender)
+    }
+
+    private fun showScreenLocally(factory: PeerConnectionFactory, videoSource: VideoSource?, localMediaStream: MediaStream?) {
+        localVideoTrack = factory.createVideoTrack(SCREEN_TRACK_ID, videoSource).apply { setEnabled(true) }
+        localMediaStream?.addTrack(localVideoTrack)
+        localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.addSink(it) } }
+    }
+
+    private fun removeLocalSurfaceRenderers() {
+        localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.removeSink(it) } }
+    }
+
+    private fun startCapturingScreen(videoCapturer: VideoCapturer, videoSource: VideoSource) {
+        val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext)
+        videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver)
+        videoCapturer.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps)
+    }
+
+    /**
+     * Returns true if the user is sharing the screen, false otherwise.
+     */
+    fun isSharingScreen(): Boolean {
+        return localVideoTrack?.enabled().orFalse() && localVideoTrack?.id() == SCREEN_TRACK_ID
     }
 
     private suspend fun release() {