diff --git a/matrix-sdk-android/src/debug/java/org/matrix/android/sdk/internal/database/RealmDebugTools.kt b/matrix-sdk-android/src/debug/java/org/matrix/android/sdk/internal/database/RealmDebugTools.kt deleted file mode 100644 index e5f4af2377..0000000000 --- a/matrix-sdk-android/src/debug/java/org/matrix/android/sdk/internal/database/RealmDebugTools.kt +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020 The Matrix.org Foundation C.I.C. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.matrix.android.sdk.internal.database - -import org.matrix.android.sdk.internal.crypto.store.db.model.CrossSigningInfoEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoRoomEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.DeviceInfoEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.GossipingEventEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.IncomingGossipingRequestEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.KeyInfoEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.KeysBackupDataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OutgoingGossipingRequestEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.TrustLevelEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity -import io.realm.Realm -import io.realm.RealmConfiguration -import io.realm.kotlin.where -import timber.log.Timber - -object RealmDebugTools { - /** - * Log info about the crypto DB - */ - fun dumpCryptoDb(realmConfiguration: RealmConfiguration) { - Realm.getInstance(realmConfiguration).use { - Timber.d("Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}") - - val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) } - Timber.d("Realm encryption key : $key") - - // Check if we have data - Timber.e("Realm is empty: ${it.isEmpty}") - - Timber.d("Realm has CryptoMetadataEntity: ${it.where().count()}") - Timber.d("Realm has CryptoRoomEntity: ${it.where().count()}") - Timber.d("Realm has DeviceInfoEntity: ${it.where().count()}") - Timber.d("Realm has KeysBackupDataEntity: ${it.where().count()}") - Timber.d("Realm has OlmInboundGroupSessionEntity: ${it.where().count()}") - Timber.d("Realm has OlmSessionEntity: ${it.where().count()}") - Timber.d("Realm has UserEntity: ${it.where().count()}") - Timber.d("Realm has KeyInfoEntity: ${it.where().count()}") - Timber.d("Realm has CrossSigningInfoEntity: ${it.where().count()}") - Timber.d("Realm has TrustLevelEntity: ${it.where().count()}") - Timber.d("Realm has GossipingEventEntity: ${it.where().count()}") - Timber.d("Realm has IncomingGossipingRequestEntity: ${it.where().count()}") - Timber.d("Realm has OutgoingGossipingRequestEntity: ${it.where().count()}") - Timber.d("Realm has MyDeviceLastSeenInfoEntity: ${it.where().count()}") - } - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt index f8dc906502..56609610f1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt @@ -238,4 +238,9 @@ interface Session : } val sharedSecretStorageService: SharedSecretStorageService + + /** + * Maintenance API, allows to print outs info on DB size to logcat + */ + fun logDbUsageInfo() } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt index 34be1b8d05..0eefca1b4c 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt @@ -155,4 +155,6 @@ interface CryptoService { // For testing shared session fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent? + + fun logDbUsageInfo() } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt index 326eac8f91..d3a3fd9fbd 100755 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt @@ -314,6 +314,10 @@ internal class DefaultCryptoService @Inject constructor( } // Just update fetchDevicesList(NoOpMatrixCallback()) + + cryptoCoroutineScope.launch(coroutineDispatchers.crypto) { + cryptoStore.tidyUpDataBase() + } } fun ensureDevice() { @@ -1291,6 +1295,11 @@ internal class DefaultCryptoService @Inject constructor( override fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent? { return cryptoStore.getWithHeldMegolmSession(roomId, sessionId) } + + override fun logDbUsageInfo() { + cryptoStore.logDbUsageInfo() + } + /* ========================================================================================== * For test only * ========================================================================================== */ diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt index 72d541d4df..9a9f645b49 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt @@ -456,4 +456,6 @@ internal interface IMXCryptoStore { fun setDeviceKeysUploaded(uploaded: Boolean) fun getDeviceKeysUploaded(): Boolean + fun tidyUpDataBase() + fun logDbUsageInfo() } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt index 5d19e6d607..72274aa70a 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt @@ -87,6 +87,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.query.get import org.matrix.android.sdk.internal.crypto.store.db.query.getById import org.matrix.android.sdk.internal.crypto.store.db.query.getOrCreate import org.matrix.android.sdk.internal.database.mapper.ContentMapper +import org.matrix.android.sdk.internal.database.tools.RealmDebugTools import org.matrix.android.sdk.internal.di.CryptoDatabase import org.matrix.android.sdk.internal.di.DeviceId import org.matrix.android.sdk.internal.di.MoshiProvider @@ -1666,4 +1667,48 @@ internal class RealmCryptoStore @Inject constructor( result } } + + /** + * Some entries in the DB can get a bit out of control with time + * So we need to tidy up a bit + */ + override fun tidyUpDataBase() { + val prevWeekTs = System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1_000 + doRealmTransaction(realmConfiguration) { realm -> + + // Only keep one week history + realm.where() + .lessThan(IncomingGossipingRequestEntityFields.LOCAL_CREATION_TIMESTAMP, prevWeekTs) + .findAll().let { + Timber.i("## Crypto Clean up ${it.size} IncomingGossipingRequestEntity") + it.deleteAllFromRealm() + } + + // Clean the cancelled ones? + realm.where() + .equalTo(OutgoingGossipingRequestEntityFields.REQUEST_STATE_STR, OutgoingGossipingRequestState.CANCELLED.name) + .equalTo(OutgoingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name) + .findAll().let { + Timber.i("## Crypto Clean up ${it.size} OutgoingGossipingRequestEntity") + it.deleteAllFromRealm() + } + + // Only keep one week history + realm.where() + .lessThan(GossipingEventEntityFields.AGE_LOCAL_TS, prevWeekTs) + .findAll().let { + Timber.i("## Crypto Clean up ${it.size} GossipingEventEntityFields") + it.deleteAllFromRealm() + } + + // Can we do something for WithHeldSessionEntity? + } + } + + /** + * Prints out database info + */ + override fun logDbUsageInfo() { + RealmDebugTools(realmConfiguration).logInfo("Crypto") + } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/tools/RealmDebugTools.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/tools/RealmDebugTools.kt new file mode 100644 index 0000000000..103e84dea6 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/tools/RealmDebugTools.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.database.tools + +import io.realm.Realm +import io.realm.RealmConfiguration +import org.matrix.android.sdk.BuildConfig +import timber.log.Timber + +internal class RealmDebugTools( + private val realmConfiguration: RealmConfiguration +) { + /** + * Log info about the DB + */ + fun logInfo(baseName: String) { + buildString { + append("\n$baseName Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}") + + if (BuildConfig.LOG_PRIVATE_DATA) { + val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) } + append("\n$baseName Realm encryption key : $key") + } + + Realm.getInstance(realmConfiguration).use { realm -> + // Check if we have data + separator() + separator() + append("\n$baseName Realm is empty: ${realm.isEmpty}") + var total = 0L + val maxNameLength = realmConfiguration.realmObjectClasses.maxOf { it.simpleName.length } + realmConfiguration.realmObjectClasses.forEach { modelClazz -> + val count = realm.where(modelClazz).count() + total += count + append("\n$baseName Realm - count ${modelClazz.simpleName.padEnd(maxNameLength)} : $count") + } + separator() + append("\n$baseName Realm - total count: $total") + separator() + separator() + } + } + .let { Timber.i(it) } + } + + private fun StringBuilder.separator() = append("\n==============================================") +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt index 7e182525a9..25345e953c 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt @@ -59,6 +59,7 @@ import org.matrix.android.sdk.api.session.user.UserService import org.matrix.android.sdk.api.session.widgets.WidgetService import org.matrix.android.sdk.internal.auth.SessionParamsStore import org.matrix.android.sdk.internal.crypto.DefaultCryptoService +import org.matrix.android.sdk.internal.database.tools.RealmDebugTools import org.matrix.android.sdk.internal.di.SessionDatabase import org.matrix.android.sdk.internal.di.SessionId import org.matrix.android.sdk.internal.di.UnauthenticatedWithCertificate @@ -197,7 +198,7 @@ internal class DefaultSession @Inject constructor( override fun close() { assert(isOpen) stopSync() - // timelineEventDecryptor.destroy() + // timelineEventDecryptor.destroy() uiHandler.post { lifecycleObservers.forEach { it.onStop() } } @@ -284,4 +285,8 @@ internal class DefaultSession @Inject constructor( override fun toString(): String { return "$myUserId - ${sessionParams.deviceId}" } + + override fun logDbUsageInfo() { + RealmDebugTools(realmConfiguration).logInfo("Session") + } } diff --git a/vector/src/main/java/im/vector/app/features/rageshake/BugReporter.kt b/vector/src/main/java/im/vector/app/features/rageshake/BugReporter.kt index 95fe16ea51..96248187aa 100755 --- a/vector/src/main/java/im/vector/app/features/rageshake/BugReporter.kt +++ b/vector/src/main/java/im/vector/app/features/rageshake/BugReporter.kt @@ -212,20 +212,20 @@ class BugReporter @Inject constructor( } activeSessionHolder.getSafeActiveSession() - ?.takeIf { !mIsCancelled && withKeyRequestHistory } - ?.cryptoService() - ?.getGossipingEvents() - ?.let { GossipingEventsSerializer().serialize(it) } - ?.toByteArray() - ?.let { rawByteArray -> - File(context.cacheDir.absolutePath, KEY_REQUESTS_FILENAME) - .also { - it.outputStream() + ?.takeIf { !mIsCancelled && withKeyRequestHistory } + ?.cryptoService() + ?.getGossipingEvents() + ?.let { GossipingEventsSerializer().serialize(it) } + ?.toByteArray() + ?.let { rawByteArray -> + File(context.cacheDir.absolutePath, KEY_REQUESTS_FILENAME) + .also { + it.outputStream() .use { os -> os.write(rawByteArray) } - } - } - ?.let { compressFile(it) } - ?.let { gzippedFiles.add(it) } + } + } + ?.let { compressFile(it) } + ?.let { gzippedFiles.add(it) } var deviceId = "undefined" var userId = "undefined" @@ -446,6 +446,10 @@ class BugReporter @Inject constructor( */ fun openBugReportScreen(activity: FragmentActivity, forSuggestion: Boolean = false) { screenshot = takeScreenshot(activity) + activeSessionHolder.getSafeActiveSession()?.let { + it.logDbUsageInfo() + it.cryptoService().logDbUsageInfo() + } val intent = Intent(activity, BugReportActivity::class.java) intent.putExtra("FOR_SUGGESTION", forSuggestion) diff --git a/vector/src/main/java/im/vector/app/features/rageshake/VectorFileLogger.kt b/vector/src/main/java/im/vector/app/features/rageshake/VectorFileLogger.kt index f26ffdd700..a279c16cb1 100644 --- a/vector/src/main/java/im/vector/app/features/rageshake/VectorFileLogger.kt +++ b/vector/src/main/java/im/vector/app/features/rageshake/VectorFileLogger.kt @@ -102,8 +102,8 @@ class VectorFileLogger @Inject constructor(val context: Context, private val vec return if (vectorPreferences.labAllowedExtendedLogging()) { false } else { - // Exclude debug and verbose logs - priority <= Log.DEBUG + // Exclude verbose logs + priority < Log.DEBUG } }