rei-forks/element-android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5911 from vector-im/feature/ons/voip_screen_sharing

Browse Source 
Screen sharing over WebRTC
This commit is contained in:
Onuray Sahin 2022-05-10 13:06:33 +03:00 committed by GitHub
commit 185cd316c9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 132 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/5911.feature Normal file
View File

@ -0,0 +1 @@
Screen sharing over WebRTC

2
vector/src/main/java/im/vector/app/features/VectorFeatures.kt
View File

@ -44,5 +44,5 @@ class DefaultVectorFeatures : VectorFeatures {
override fun isOnboardingPersonalizeEnabled() = false
override fun isOnboardingCombinedRegisterEnabled() = false
override fun isLiveLocationEnabled(): Boolean = false
override fun isScreenSharingEnabled(): Boolean = false
override fun isScreenSharingEnabled(): Boolean = true
}

2
vector/src/main/java/im/vector/app/features/call/CallControlsView.kt
View File

@ -89,6 +89,8 @@ class CallControlsView @JvmOverloads constructor(
views.videoToggleIcon.setImageResource(R.drawable.ic_video_off)
views.videoToggleIcon.contentDescription = resources.getString(R.string.a11y_start_camera)
}
views.videoToggleIcon.isEnabled = !state.isSharingScreen
views.videoToggleIcon.alpha = if (state.isSharingScreen) 0.5f else 1f
when (callState) {
is CallState.LocalRinging -> {

42
vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt
View File

@ -24,6 +24,7 @@ import android.content.Intent
import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP
import android.content.res.Configuration
import android.graphics.Color
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.os.Build
import android.os.Bundle
@ -32,6 +33,7 @@ import android.util.Rational
import android.view.MenuItem
import android.view.View
import android.view.WindowManager
import androidx.activity.result.ActivityResult
import androidx.annotation.StringRes
import androidx.core.content.ContextCompat
import androidx.core.content.getSystemService
@ -76,6 +78,7 @@ import org.matrix.android.sdk.api.session.call.TurnServerResponse
import org.matrix.android.sdk.api.session.room.model.call.EndCallReason
import org.webrtc.EglBase
import org.webrtc.RendererCommon
import org.webrtc.ScreenCapturerAndroid
import timber.log.Timber
import javax.inject.Inject
@ -161,6 +164,9 @@ class VectorCallActivity : VectorBaseActivity<ActivityCallBinding>(), CallContro
}
}
}
// Bind to service in case of user killed the app while there is an ongoing call
bindToScreenCaptureService()
}
override fun onNewIntent(intent: Intent?) {
@ -636,18 +642,40 @@ class VectorCallActivity : VectorBaseActivity<ActivityCallBinding>(), CallContro
private val screenSharingPermissionActivityResultLauncher = registerStartForActivityResult { activityResult ->
if (activityResult.resultCode == Activity.RESULT_OK) {
callViewModel.handle(VectorCallViewActions.StartScreenSharing)
// We need to start a foreground service with a sticky notification during screen sharing
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
ContextCompat.startForegroundService(
this,
Intent(this, ScreenCaptureService::class.java)
)
screenCaptureServiceConnection.bind()
// We need to start a foreground service with a sticky notification during screen sharing
startScreenSharingService(activityResult)
} else {
startScreenSharing(activityResult)
}
}
}
private fun startScreenSharing(activityResult: ActivityResult) {
val videoCapturer = ScreenCapturerAndroid(activityResult.data, object : MediaProjection.Callback() {
override fun onStop() {
Timber.i("User revoked the screen capturing permission")
}
})
callViewModel.handle(VectorCallViewActions.StartScreenSharing(videoCapturer))
}
private fun startScreenSharingService(activityResult: ActivityResult) {
ContextCompat.startForegroundService(
this,
Intent(this, ScreenCaptureService::class.java)
)
bindToScreenCaptureService(activityResult)
}
private fun bindToScreenCaptureService(activityResult: ActivityResult? = null) {
screenCaptureServiceConnection.bind(object : ScreenCaptureServiceConnection.Callback {
override fun onServiceConnected() {
activityResult?.let { startScreenSharing(it) }
}
})
}
private fun handleShowScreenSharingPermissionDialog() {
getSystemService<MediaProjectionManager>()?.let {
navigator.openScreenSharingPermissionDialog(it.createScreenCaptureIntent(), screenSharingPermissionActivityResultLauncher)

3
vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt
View File

@ -19,6 +19,7 @@ package im.vector.app.features.call
import im.vector.app.core.platform.VectorViewModelAction
import im.vector.app.features.call.audio.CallAudioManager
import im.vector.app.features.call.transfer.CallTransferResult
import org.webrtc.VideoCapturer
sealed class VectorCallViewActions : VectorViewModelAction {
object EndCall : VectorCallViewActions()
@ -41,5 +42,5 @@ sealed class VectorCallViewActions : VectorViewModelAction {
data class CallTransferSelectionResult(val callTransferResult: CallTransferResult) : VectorCallViewActions()
object TransferCall : VectorCallViewActions()
object ToggleScreenSharing : VectorCallViewActions()
object StartScreenSharing : VectorCallViewActions()
data class StartScreenSharing(val videoCapturer: VideoCapturer) : VectorCallViewActions()
}

25
vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt
View File

@ -145,9 +145,10 @@ class VectorCallViewModel @AssistedInject constructor(
override fun onCallEnded(callId: String) {
withState { state ->
if (state.otherKnownCallInfo?.callId == callId) {
setState { copy(otherKnownCallInfo = null) }
setState { copy(otherKnownCallInfo = null, isSharingScreen = false) }
}
}
_viewEvents.post(VectorCallViewEvents.StopScreenSharingService)
}
override fun onCurrentCallChange(call: WebRtcCall?) {
@ -156,9 +157,10 @@ class VectorCallViewModel @AssistedInject constructor(
}
}
override fun onAudioDevicesChange() {
val currentSoundDevice = callManager.audioManager.selectedDevice ?: return
if (currentSoundDevice == CallAudioManager.Device.Phone) {
override fun onAudioDevicesChange() = withState { state ->
val currentSoundDevice = callManager.audioManager.selectedDevice ?: return@withState
val webRtcCall = callManager.getCallById(state.callId)
if (webRtcCall != null && shouldActivateProximitySensor(webRtcCall)) {
proximityManager.start()
} else {
proximityManager.stop()
@ -205,7 +207,7 @@ class VectorCallViewModel @AssistedInject constructor(
callManager.addListener(callManagerListener)
webRtcCall.addListener(callListener)
val currentSoundDevice = callManager.audioManager.selectedDevice
if (currentSoundDevice == CallAudioManager.Device.Phone) {
if (shouldActivateProximitySensor(webRtcCall)) {
proximityManager.start()
}
setState {
@ -224,13 +226,18 @@ class VectorCallViewModel @AssistedInject constructor(
formattedDuration = webRtcCall.formattedDuration(),
isHD = webRtcCall.mxCall.isVideoCall && webRtcCall.currentCaptureFormat() is CaptureFormat.HD,
canOpponentBeTransferred = webRtcCall.mxCall.capabilities.supportCallTransfer(),
transferee = computeTransfereeState(webRtcCall.mxCall)
transferee = computeTransfereeState(webRtcCall.mxCall),
isSharingScreen = webRtcCall.isSharingScreen()
)
}
updateOtherKnownCall(webRtcCall)
}
}
private fun shouldActivateProximitySensor(webRtcCall: WebRtcCall): Boolean {
return callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone && !webRtcCall.isSharingScreen()
}
private fun WebRtcCall.extractCallInfo(): VectorCallViewState.CallInfo {
val assertedIdentity = this.remoteAssertedIdentity
val matrixItem = if (assertedIdentity != null) {
@ -349,7 +356,8 @@ class VectorCallViewModel @AssistedInject constructor(
handleToggleScreenSharing(state.isSharingScreen)
}
is VectorCallViewActions.StartScreenSharing -> {
call?.startSharingScreen()
call?.startSharingScreen(action.videoCapturer)
proximityManager.stop()
setState {
copy(isSharingScreen = true)
}
@ -366,6 +374,9 @@ class VectorCallViewModel @AssistedInject constructor(
_viewEvents.post(
VectorCallViewEvents.StopScreenSharingService
)
if (callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone) {
proximityManager.start()
}
} else {
_viewEvents.post(
VectorCallViewEvents.ShowScreenSharingPermissionDialog

15
vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt
View File

@ -27,11 +27,20 @@ class ScreenCaptureServiceConnection @Inject constructor(
private val context: Context
) : ServiceConnection {
interface Callback {
fun onServiceConnected()
}
private var isBound = false
private var screenCaptureService: ScreenCaptureService? = null
private var callback: Callback? = null
fun bind() {
if (!isBound) {
fun bind(callback: Callback) {
this.callback = callback
if (isBound) {
callback.onServiceConnected()
} else {
Intent(context, ScreenCaptureService::class.java).also { intent ->
context.bindService(intent, this, 0)
}
@ -45,10 +54,12 @@ class ScreenCaptureServiceConnection @Inject constructor(
override fun onServiceConnected(className: ComponentName, binder: IBinder) {
screenCaptureService = (binder as ScreenCaptureService.LocalBinder).getService()
isBound = true
callback?.onServiceConnected()
}
override fun onServiceDisconnected(className: ComponentName) {
isBound = false
screenCaptureService = null
callback = null
}
}

66
vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt
View File

@ -80,10 +80,12 @@ import org.webrtc.MediaConstraints
import org.webrtc.MediaStream
import org.webrtc.PeerConnection
import org.webrtc.PeerConnectionFactory
import org.webrtc.RtpSender
import org.webrtc.RtpTransceiver
import org.webrtc.SessionDescription
import org.webrtc.SurfaceTextureHelper
import org.webrtc.SurfaceViewRenderer
import org.webrtc.VideoCapturer
import org.webrtc.VideoSource
import org.webrtc.VideoTrack
import timber.log.Timber
@ -95,6 +97,7 @@ import kotlin.coroutines.CoroutineContext
private const val STREAM_ID = "userMedia"
private const val AUDIO_TRACK_ID = "${STREAM_ID}a0"
private const val VIDEO_TRACK_ID = "${STREAM_ID}v0"
private const val SCREEN_TRACK_ID = "${STREAM_ID}s0"
private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints()
private const val INVITE_TIMEOUT_IN_MS = 60_000L
@ -153,13 +156,16 @@ class WebRtcCall(
private var makingOffer: Boolean = false
private var ignoreOffer: Boolean = false
private var videoCapturer: CameraVideoCapturer? = null
private var videoCapturer: VideoCapturer? = null
private val availableCamera = ArrayList<CameraProxy>()
private var cameraInUse: CameraProxy? = null
private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD
private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null
private var videoSender: RtpSender? = null
private var screenSender: RtpSender? = null
private val timer = CountUpTimer(1000L).apply {
tickListener = object : CountUpTimer.TickListener {
override fun onTick(milliseconds: Long) {
@ -617,7 +623,7 @@ class WebRtcCall(
val videoTrack = peerConnectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource)
Timber.tag(loggerTag.value).v("Add video track $VIDEO_TRACK_ID to call ${mxCall.callId}")
videoTrack.setEnabled(true)
peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
videoSender = peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
localVideoSource = videoSource
localVideoTrack = videoTrack
}
@ -718,7 +724,7 @@ class WebRtcCall(
Timber.tag(loggerTag.value).v("switchCamera")
if (mxCall.state is CallState.Connected && mxCall.isVideoCall) {
val oppositeCamera = getOppositeCameraIfAny() ?: return@launch
videoCapturer?.switchCamera(
(videoCapturer as? CameraVideoCapturer)?.switchCamera(
object : CameraVideoCapturer.CameraSwitchHandler {
// Invoked on success. |isFrontCamera| is true if the new camera is front facing.
override fun onCameraSwitchDone(isFrontCamera: Boolean) {
@ -766,12 +772,60 @@ class WebRtcCall(
return currentCaptureFormat
}
fun startSharingScreen() {
// TODO. Will be handled within the next PR.
fun startSharingScreen(videoCapturer: VideoCapturer) {
val factory = peerConnectionFactoryProvider.get() ?: return
this.videoCapturer = videoCapturer
val localMediaStream = factory.createLocalMediaStream(STREAM_ID)
val videoSource = factory.createVideoSource(videoCapturer.isScreencast)
startCapturingScreen(videoCapturer, videoSource)
removeLocalSurfaceRenderers()
showScreenLocally(factory, videoSource, localMediaStream)
videoSender?.let { removeStream(it) }
screenSender = peerConnection?.addTrack(localVideoTrack, listOf(STREAM_ID))
}
fun stopSharingScreen() {
// TODO. Will be handled within the next PR.
localVideoTrack?.setEnabled(false)
screenSender?.let { removeStream(it) }
if (mxCall.isVideoCall) {
peerConnectionFactoryProvider.get()?.let { configureVideoTrack(it) }
}
updateMuteStatus()
sessionScope?.launch(dispatcher) { attachViewRenderersInternal() }
}
private fun removeStream(sender: RtpSender) {
peerConnection?.removeTrack(sender)
}
private fun showScreenLocally(factory: PeerConnectionFactory, videoSource: VideoSource?, localMediaStream: MediaStream?) {
localVideoTrack = factory.createVideoTrack(SCREEN_TRACK_ID, videoSource).apply { setEnabled(true) }
localMediaStream?.addTrack(localVideoTrack)
localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.addSink(it) } }
}
private fun removeLocalSurfaceRenderers() {
localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.removeSink(it) } }
}
private fun startCapturingScreen(videoCapturer: VideoCapturer, videoSource: VideoSource) {
val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext)
videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver)
videoCapturer.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps)
}
/**
* Returns true if the user is sharing the screen, false otherwise.
*/
fun isSharingScreen(): Boolean {
return localVideoTrack?.enabled().orFalse() && localVideoTrack?.id() == SCREEN_TRACK_ID
}
private suspend fun release() {