diff --git a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt b/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt
index 6eda1b1604..aad9e90780 100644
--- a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt
+++ b/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt
@@ -322,7 +322,9 @@ class KeysBackupTest : InstrumentedTest {
                     put(cryptoTestData.roomId, roomKeysBackupData)
                 }
         )
-        algorithm.setRecoveryKey(keyBackupCreationInfo.recoveryKey)
+        extractCurveKeyFromRecoveryKey(keyBackupCreationInfo.recoveryKey)?.also {
+            algorithm.setPrivateKey(it)
+        }
         val sessionsData = algorithm.decryptSessions(keysBackupData)
         val sessionData = sessionsData.firstOrNull()
         assertNotNull(sessionData)
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt
index 22590b97fe..0344665ed5 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt
@@ -632,7 +632,9 @@ internal class DefaultKeysBackupService @Inject constructor(
 
                 // Get backed up keys from the homeserver
                 val data = getKeys(sessionId, roomId, keysVersionResult.version)
-                algorithm?.setRecoveryKey(recoveryKey)
+                extractCurveKeyFromRecoveryKey(recoveryKey)?.also { privateKey ->
+                    algorithm?.setPrivateKey(privateKey)
+                }
                 val sessionsData = withContext(coroutineDispatchers.computation) {
                     algorithm?.decryptSessions(data)
                 }.orEmpty()
@@ -1138,7 +1140,9 @@ internal class DefaultKeysBackupService @Inject constructor(
                 // roomId -> sessionId -> MXKeyBackupData
                 val keysBackupData = KeysBackupData()
                 val recoveryKey = cryptoStore.getKeyBackupRecoveryKeyInfo()?.recoveryKey
-                algorithm?.setRecoveryKey(recoveryKey)
+                extractCurveKeyFromRecoveryKey(recoveryKey)?.also { privateKey ->
+                    algorithm?.setPrivateKey(privateKey)
+                }
                 olmInboundGroupSessionWrappers.forEach { olmInboundGroupSessionWrapper ->
                     val roomId = olmInboundGroupSessionWrapper.roomId ?: return@forEach
                     val olmInboundGroupSession = olmInboundGroupSessionWrapper.session
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAes256Algorithm.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAes256Algorithm.kt
index 87245cfc19..4a3ee60d32 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAes256Algorithm.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAes256Algorithm.kt
@@ -46,8 +46,8 @@ internal class KeysBackupAes256Algorithm(keysVersions: KeysVersionResult) : Keys
         aesAuthData = keysVersions.getAuthDataAsMegolmBackupAuthData() as MegolmBackupAes256AuthData
     }
 
-    override fun setRecoveryKey(recoveryKey: String?) {
-        privateKey = extractCurveKeyFromRecoveryKey(recoveryKey)
+    override fun setPrivateKey(privateKey: ByteArray) {
+        this.privateKey = privateKey
     }
 
     override fun encryptSession(sessionData: MegolmSessionData): JsonDict? {
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAlgorithm.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAlgorithm.kt
index 812d34984d..0e15a6eed6 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAlgorithm.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupAlgorithm.kt
@@ -27,7 +27,7 @@ internal interface KeysBackupAlgorithm {
     val authData: MegolmBackupAuthData
     val untrusted: Boolean
 
-    fun setRecoveryKey(recoveryKey: String?)
+    fun setPrivateKey(privateKey: ByteArray)
     fun encryptSession(sessionData: MegolmSessionData): JsonDict?
     fun decryptSessions(data: KeysBackupData): List<MegolmSessionData>
     fun keyMatches(privateKey: ByteArray): Boolean
diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupCurve25519Algorithm.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupCurve25519Algorithm.kt
index 627024991b..cf1abe831b 100644
--- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupCurve25519Algorithm.kt
+++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/algorithm/KeysBackupCurve25519Algorithm.kt
@@ -52,8 +52,8 @@ internal class KeysBackupCurve25519Algorithm(keysVersions: KeysVersionResult) :
         }
     }
 
-    override fun setRecoveryKey(recoveryKey: String?) {
-        privateKey = extractCurveKeyFromRecoveryKey(recoveryKey)
+    override fun setPrivateKey(privateKey: ByteArray) {
+        this.privateKey = privateKey
     }
 
     override fun encryptSession(sessionData: MegolmSessionData): JsonDict? {