Merge branch 'release/1.0.10'

AUTHORS.md

@@ -33,3 +33,8 @@ First of all, we thank all contributors who use Element and report problems on t
 We do not forget all translators, for their work of translating Element into many languages. They are also the authors of Element.
 
 Feel free to add your name below, when you contribute to the project!
+
+Name    | Matrix ID           | GitHub
+--------|---------------------|--------------------------------------
+gjpower | @gjpower:matrix.org | [gjpower](https://github.com/gjpower)
+

CHANGES.md

@@ -1,3 +1,30 @@
+Changes in Element 1.0.10 (2020-11-04)
+===================================================
+
+Improvements 🙌:
+ - Rework sending Event management (#154)
+ - New room creation screen: set topic and avatar in the room creation form (#2078)
+ - Toggle Low priority tag (#1490)
+ - Add option to send with enter (#1195)
+ - Use Hardware keyboard enter to send message (use shift-enter for new line) (#1881, #1440)
+ - Edit and remove icons are now visible on image attachment preview screen (#2294)
+ - Room profile: BigImageViewerActivity now only display the image. Use the room setting to change or delete the room Avatar
+ - Better visibility of text reactions in dark theme (#1118)
+ - Room member profile: Add action to create (or open) a DM (#2310)
+ - Prepare changelog for F-Droid (#2296)
+ - Add graphic resources for F-Droid (#812, #2220)
+ - Highlight text in the body of the displayed result (#2200)
+ - Considerably faster QR-code bitmap generation (#2331)
+
+Bugfix 🐛:
+ - Fixed ringtone handling (#2100 & #2246)
+ - Messages encrypted with no way to decrypt after SDK update from 0.18 to 1.0.0 (#2252)
+ - Incoming call continues to ring if call is answered on another device (#1921)
+ - Search Result | scroll jumps after pagination (#2238)
+ - Badly formatted mentions in body (#1506)
+ - KeysBackup: Avoid using `!!` (#2262)
+ - Two elements in the task switcher (#2299)
+
 Changes in Element 1.0.9 (2020-10-16)
 ===================================================
 

build.gradle

@@ -65,9 +65,8 @@ allprojects {
     tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {
         // Warnings are potential errors, so stop ignoring them
         // You can override by passing `-PallWarningsAsErrors=false` in the command line
-        kotlinOptions.allWarningsAsErrors = project.properties['allWarningsAsErrors']?.toBoolean() ?: true
+        kotlinOptions.allWarningsAsErrors = project.getProperties().getOrDefault("allWarningsAsErrors", "true").toBoolean()
     }
-
 }
 
 task clean(type: Delete) {

fastlane/metadata/android/bg/full_description.txt

@@ -0,0 +1,30 @@
+Element е приложение от нов тип за съобщения и сътрудничество:
+
+1. Дава Ви контрол, за да запазите поверителността си
+2. Позволява ви да комуникирате с всеки в мрежата на Matrix и дори извън него, като се интегрира с приложения като Slack
+3. Предпазва ви от реклами, изтичане на данни и търговско следене
+4. Защитава ви чрез шифроване от край до край, с кръстосано подписване, за да проверите другите
+
+Element е напълно различен от другите приложения за съобщения и сътрудничество, понеже е децентрализиран и с отворен код.
+
+Element ви позволява да го хоствате самостоятелно - или да изберете хост - така че да имате поверителност, собственост и контрол върху Вашите данни и разговори. Дава ви достъп до отворена мрежа, така че комуникацията Ви не е ограничена до потребителите на Element. И е много сигурно.
+
+Element е в състояние да направи всичко това, защото работи върху Matrix - стандартът за отворена, децентрализирана комуникация.
+
+Element ви дава контрол, като ви позволява да изберете кой да хоства Вашите разговори. От приложението Element можете да изберете хостване по различни начини:
+
+1. Вземете безплатен профил на публичния сървър на matrix.org, хостван от разработчиците на Matrix, или изберете от хиляди публични сървъри, хоствани от доброволци
+2. Самостоятелно хоствайте профила си, като пуснете сървър на собствен хардуер
+3. Регистрирайте се за профил на персонализиран сървър, като се абонирате за хостинг платформата Element Matrix Services
+
+<b>Защо да изберете Element?</b>
+
+<b>ПРИТЕЖАВАЙТЕ ДАННИТЕ СИ</b>: Вие решавате къде да съхранявате вашите данни и съобщения. Вие ги притежавате и контролирате, а не някаква МЕГАКОРПОРАЦИЯ, която складира вашите данни или дава достъп на трети страни.
+
+<b>ОТВОРЕНИ СЪОБЩЕНИЯ И СЪТРУДНИЧЕСТВО</b>: Можете да разговаряте с всеки друг в мрежата на Matrix, независимо дали използва Element или друго приложение на Matrix и дори ако използва различна система за съобщения като Slack, IRC or XMPP.
+
+<b>СВРЪХ СИГУРНО</b>: Реално шифроване от край до край (само тези в разговора могат да дешифрират съобщения) и кръстосано подписване за проверка на устройствата на участниците в разговора.
+
+<b>ПЪЛНА КОМУНИКАЦИЯ</b>: Съобщения, гласови и видео разговори, споделяне на файлове, споделяне на екран и цял куп интеграции, ботове и джаджи. Изграждайте стаи, общности, поддържайте връзка и направете нещата завършени.
+
+<b>НАВСЯКЪДЕ КЪДЕТО СТЕ</b>: Поддържайте връзка, където и да сте, с напълно синхронизирана история на съобщенията на всичките ви устройства и чрез web на https://app.element.io.

fastlane/metadata/android/bg/short_description.txt

@@ -0,0 +1 @@
+Сигурен децентрализиран чат и VoIP. Пазете данните си от външни лица.

fastlane/metadata/android/bg/title.txt

@@ -0,0 +1 @@
+Element (предишен Riot.im)

fastlane/metadata/android/en-US/changelogs/40100100.txt

@@ -0,0 +1,2 @@
+This new version mainly contains bug fixes and improvements. Sending a message is now much faster.
+Full changelog: https://github.com/vector-im/element-android/releases/tag/v1.0.10

fastlane/metadata/android/eo/full_description.txt

@@ -0,0 +1,30 @@
+Element estas nova speco de mesaĝilo kaj kunlaborilo, kiu:
+
+1. Lasas vin regi vian komunikadon por protekti vian privatecon
+2. Lasas vin komuniki kun ĉiu ajn en la reto de Matrix, kaj eĉ pliaj, per interkompreno kun aplikaĵoj kiel ekzemple Slack
+3. Protektas vin de reklamoj, kolektado de datumoj, kaj muritaj ĝardenoj
+4. Sekurigas vian komunikadon per tutvoja ĉifrado, kun la eblo kontroli aliajn per delegaj subskriboj
+
+Element malsamas al aliaj mesaĝiloj kaj kunlaboriloj, ĉar ĝi estas sencentra kaj malfermitkoda.
+
+Element lasas vin gastigi vian propran servilon, aŭ elekti servilon, kiu plaĉas al vi, por ke vi ne perdu privatecon, kaj por ke vi daŭre regu kaj posedu viajn datumojn kaj interparolojn. Ĝi donas al vi aliron al malfermita reto; por ke via interparolado ne estu limigita nur al aliaj uzantoj de Element. Kaj ĝi estas tre sekura.
+
+Element povas fari ĉi ĉion, ĉar ĝi funkcias per Matrix – publika normo por malfermita, sencentra komunikado.
+
+Element lasas vi elekti, kiu gastigos viajn interparolojn. Per la aplikaĵo Element, vi povas elekti diversajn specojn de gastigado:
+
+1. Akiri senpagan konton ĉe la publika servilo matrix.org, gastigata de la programistoj de Matrix, aŭ elekti unu el miloj da publikaj serviloj, gastigataj de volontuloj
+2. Memgastiĝi per via propra servilo ĉe via propra aparato
+3. Registriĝi ĉe propra servilo per simpla pagaliĝo al la gastiga platformo «Element Matrix Services»
+
+<b>Kial Element?</b>
+
+<b>POSEDU VIAJN DATUMOJN</b>: Vi decidu, kie vi tenu viajn datumojn kaj mesaĝojn. Vi posedas kaj regas ilin, ne iu granda komerca firmao, kiu kolektas viajn datumojn aŭ donas aliron al aliuloj.
+
+<b>MALFERMAJ MESAĜADO KAJ KUNLABORADO</b>: Vi povas babili kun ĉiu alia en la reto de Matrix, ĉu ĝi uzas Elementon aŭ alian aplikaĵon de Matrix, kaj eĉ se ĝi uzas tute alian mesaĝilon, kiel ekzemple Slack, IRC, aŭ XMPP.
+
+<b>TRE SEKURA</b>: Vera tutvoja ĉifrado (nur la interparolantoj povas malĉifri siajn mesaĝojn), kaj delegaj subskriboj por kontroli la aparatojn de partoprenantoj.
+
+<b>SENMANKA KOMUNIKADO</b>: Mesaĝoj, voĉvokoj kaj vidvokoj, havigado de dosieroj, ekrano, kaj multaj diversaj kunigoj, robotoj kaj fenestraĵoj. Kreu ĉambrojn, komunumojn, komuniku kaj kunlaboru.
+
+<b>ĈIE KUN VI</b>: Tenu vin ĝisdata per historio de mesaĝoj plene spegulita trans ĉiuj viaj aparatoj, kaj sur la reto per https://app.element.io.

fastlane/metadata/android/eo/short_description.txt

@@ -0,0 +1 @@
+Sekura kaj sencentrigita vokado kaj babilado. Tenu viajn datumojn sekuraj.

fastlane/metadata/android/eo/title.txt

@@ -0,0 +1 @@
+Element (antaŭe Riot.im)

fastlane/metadata/android/et/full_description.txt

@@ -0,0 +1,30 @@
+Element on uut tüüpi suhtlus- ja koostöörakendus, mis:
+
+1. Võimaldab täielikku kontrolli privaatsuse üle
+2. Võimaldab suhelda kõigiga Matrixi võrgus ja isegi väljaspool seda, olles integreeritud selliste rakendustega nagu Slack
+3. Kaitseb sind reklaamide ja andmekogumise eest
+4. Tagab turvalisuse läbiva krüptimise abil, kasutades risttunnustamist vestlejate tuvastamiseks
+
+Element erineb täielikult teistest sõnumside- ja koostöörakendustest, kuna see on detsentraliseeritud ja avatud lähtekoodiga.
+
+Element võimaldab ise hostida - või valida hosti -, et oleks tagatud privaatsus ja kontroll oma andmete ja vestluste üle. Element annab ka juurdepääsu avatud võrgule, nii et te ei pea vaid Elemendi kasutajatega rääkima. Ning kogu süsteem on väga turvaline.
+
+Element töötab Matrixil - avatud, detsentraliseeritud suhtlus-standardil.
+
+Võimaldades valida, kes vestlusi korraldab, annab Element annab kontrolli sinule. Rakendust Element saad kasutada mitmel viisil.
+
+1. Tasuta konto Matrixi arendajate hostitud avalikus serveris matrix.org või vali tuhandete avalike serverite hulgast, mida haldavad vabatahtlikud
+2. Hosti oma kontot ise, paigaldades serveri oma riistvarale
+3. Registreeruge serveris olevale kontole, tellides Element Matrix Services teenuseplatvormi
+
+<b> Miks valida element? </b>
+
+<b> KONTROLL ANDMETE ÜLE</b>: otsustad ise, kus oma andmeid ja sõnumeid hoida. Need kuuluvad sulle ja sinu käes on kontroll, mitte mõne MEGAFIRMA käes, mis andmeid oma kasuks kaevandab või kolmandatele isikutele juurdepääsu annab.
+
+<b> AVATUD SUHTLUS JA KOOSTÖÖ </b>: saad vestelda kõigi teistega Matrixi võrgus, olenemata sellest, kas nad kasutavad Elementi või mõnda muud Matrixi rakendust, ja isegi kui nad kasutavad teistsugust suhtlussüsteemi nagu Slack, IRC või XMPP.
+
+<b> ÜLITURVALINE </b>: tõeline läbiv krüptimine (ainult vestluses osalejad saavad sõnumeid lugeda) ja risttunnustamine vestluses osalejate tuvastamiseks.
+
+<b> KÕIK SUHTLUSVÕIMALUSED</b>: sõnumid, hääl- ja videokõned, failide jagamine, ekraani jagamine ja terve hulk lõiminguid, roboteid ja vidinaid. Loo tubasid, kogukondi, hoia ühendust ja saa asjad aetud.
+
+<b> KÕIKJAL, KUS VIIBITE</b>: saad suhelda kõigis oma seadmetes ja ka veebis aadressil https://app.element.io ning sealjuures täielikult sünkroonitud sõnumite ajalooga.

fastlane/metadata/android/fa/full_description.txt

@@ -0,0 +1,30 @@
+المنت گونه‌ای جدید از کاره‌های پیام‌رسانی و همکاری است که:
+
+۱. کنترل محرمانگیتان را در دست خودتان می‌گذارد
+۲. می‌گذارد با هرکسی در شبکهٔ ماتریکس و حتا فراتر از آن، ارتباط برقرار کنید
+۳. از شما در برابر تبلیغات، داده‌کاوری و دیوارهای پرداختی، محافظت می‌کند
+۴. با رمزنگاری سرتاسری با ورود چندگانه، امنتان می‌کند
+
+المنت به خاطر نامتمرکز و نرم‌افزار آزاد بودن، کاملاً با دیگر کاره‌های پیام‌رسانی و همکاری، فرق دارد.
+
+المنت می‌گذارد خودمیزبانی کرده یا میزبانی برگزینید که امنیت، مالکیت و واپایش داده‌ها و گفت‌وگوهایتان را در اختیار داشته باشید. این کاره شما را به شبکه‌ای باز و شدیداً امن وصل کرده تا مجبور نباشید فقط با دیگر کاربران المنت صحبت کنید.
+
+المنت می‌تواند همهٔ این کارها را بکند، چرا که روی ماتریکس، استانداردی برای گفت‌وگوی باز و نامتمرکز عمل می‌کند.
+
+المنت با اجازه برای گزینش کسی که گفت‌وگوهایتان را میزبانی می‌کند، کنترل را به شما می‌دهد. با کارهٔ المنت، می‌توانید برگزینید که به روش‌های مختلفی میزبانی شوید:
+
+۱. گرفتن حسابی رایگان روی کارساز عمومی matrix.org که به دست توسعه‌دهندگان ماتریکس میزبانی می‌شود، یا گرینش از میان هزاران کارساز عمومی میزبانی‌شده به دست داوطلبان
+۲. خودمیزبانی حسابتان با اجرای کراسازی روی سخت‌افزار خودتان
+۳. ثبت‌نام برای حسابی روی یک کارساز سفارشی با اشتراک در بن‌یازهٔ میزبانی خدمات ماتریکس المنت
+
+<b>چرا المنت را برگزینیم؟</b>
+
+<b>مالک داده‌هایتان باشید</b>: خوتان تصمیم می‌گیرید که داده‌ها و پیام‌هایتان را کجا نگه دارید. شما صاحبشان هستید و واپایششان می‌کنید، نه شرکت‌های بزرگی که داده‌هایتان را کاویده و به شرکت‌های دیگر دسترسی می‌دهند.
+
+<b>پیام‌رسانی و همکاری باز</b>: می‌توانید با هرکسی در شبکهٔ ماتریکس گپ بزنید، چه از المنت استفاده کنند و چه از هر کارهٔ ماتریکس دیگری؛ و حتا اگر از سامانهٔ پیام‌رسانی متفاوتی مثل اسلک، آی‌آرسی یا جبر استفاده کنند.
+
+<b>فوق امن</b>: رمزنگاری سرتاسری واقعی (فقط کسانی که در گفت‌وگو هستند،‌می‌توانند پیام‌ها را رمزگشایی کنند) و ورود چندگانه برای تأیید هویت افزاره‌های شرکت‌کنندگان در گفت‌وگو.
+
+<b>ارتباط کامل</b>: پیام‌رسانی، تماس‌های صوتی و تصویری،‌هم‌رسانی پرونده، هم‌رسانی صفحه و یه عالمه یکپارچگی، بات و ابزارک. اتاق و اجتماع ساخته، در دسترس بوده و کارها را انجام دهید.
+
+<b>هرجا که هستید</b>: هر کجا که هستید، با هم‌گام سازی کامل تاریخچهٔ پیام‌ها بین همهٔ افزاره‌هایتان و وب روی https://app.element.io در دسترس باشید.

fastlane/metadata/android/fa/short_description.txt

@@ -0,0 +1 @@
+گپ و تماس نامتمرکز امن. داده‌هایتان را از شرکت‌ها امن نگه دارید.

fastlane/metadata/android/fa/title.txt

@@ -0,0 +1 @@
+المنت (ریوت سابق)

fastlane/metadata/android/fr/full_description.txt

@@ -0,0 +1,30 @@
+Element est une nouvelle application de messagerie et de collaboration qui :
+
+1) Vous place aux commandes de votre vie privée
+2) Vous permet de communiquer avec n'importe qui du réseau Matrix, et plus encore par des intégrations d'autres applications comme Slack ou Discord
+3) Vous protège de la publicité et de la collecte de données
+4) Vous sécurise grâce à du chiffrement bout-à-bout, avec de la signature croisée pour authentifier les autres utilisateurs
+
+Element est complètement différent des autres applications de messagerie et de collaboration puisque l'application est décentralisée et open-source.
+
+Element vous permet d'héberger vous-même -ou de choisir un hôte- vous permettant d'assurer votre vie privée, la propriété et le contrôle de vos données et de vos conversations. Cela vous offre l'accès à un réseau ouvert, vous n'êtes donc pas condamné à parler à d'autres utilisateurs d'Element seulement. Et c'est très sécurisé.
+
+Element peut faire tout ça car il est basé sur Matrix, le protocole standard pour la communication ouverte et décentralisée.
+
+Element vous donne le contrôle en vous laissant choisir qui héberge vos conversations. Depuis l'application Element, vous pouvez choisir votre hôte de différentes manières :
+
+1) Créer un compte gratuit sur le serveur public matrix.org hébergé par les développeurs de Matrix, ou choisir parler les milliers de serveurs public hébergés par des bénévoles
+2) Héberger vous-même votre compte en installant un serveur sur votre propre machine
+3) Créer un compte sur un serveur personnalisé en souscrivant sur la plateforme d'hébergement « Element Matrix Services » (EMS)
+
+<b>Pourquoi choisir Element ?</b>
+
+<b>POSSÉDEZ VOS DONNÉES</b> : Vous décidez où conserver vos données et vos messages. Vous les possédez et vous les contrôlez, et non une MEGACORP qui mine vos données ou les donnent à des tiers
+
+<b>UNE MESSAGERIE OUVERTE ET COLLABORATIVE</b> : Vous pouvez discuter avec n'importe qui sur le réseau Matrix, qu'ils utilisent Element ou une autre application basée sur Matrix, et même s'ils utilisent un système de messagerie différent comment Slack, Discord, IRC ou XMPP.
+
+<b>SUPER SÉCURISÉ</b> : Un réel chiffrement bout-à-bout (seulement ceux deux la conversation peuvent déchiffrer les messages), et une signature croisée pour vérifier les appareils des participants de la conversation.
+
+<b>COMMUNICATION COMPLÈTE</b> : Messagerie, appels vocaux et vidéo, transfert de fichiers, partage d'écran et un tas d'intégrations, robots et widgets. Construisez des salons, des communautés, restez en contact et accomplissez de grandes choses.
+
+<b>PARTOUT OÙ VOUS ÊTES</b> : Restez connectés peu import où vous êtes avec la synchronisation complète de l'historique des messages sur tous vos appareils et sur le web sur https://app.element.io.

fastlane/metadata/android/fr/short_description.txt

@@ -0,0 +1 @@
+Chat & VoIP sûr et décentralisé. Gardez vos données en sécurité.

fastlane/metadata/android/hu/title.txt

@@ -1 +1 @@
-Element (előzőleg Riot.im)
+Element (régebben Riot.im)

fastlane/metadata/android/sv/full_description.txt

@@ -21,7 +21,7 @@ Element sätter dig i kontroll genom att låta dig välja att vara värd för di
 
 <b>ÄG DIN DATA</b>: Du väljer var du vill ha din data och dina meddelanden. Du äger den och kontrollerar den, inte nåt stort företag som samlar in din data och ger den till tredje parter.
 
-<b>ÖPPEN KOMMUNIKATION OCH ÖPPET SAMARBETE</b>: Du kan chatta med med vem som helst på Matrix-nätverket, oavsett om de använder Element eller en annan Matrix-app, och till och med om de använder ett annat meddelandesystem som Slack, IRC eller XMPP.
+<b>ÖPPEN KOMMUNIKATION OCH ÖPPET SAMARBETE</b>: Du kan chatta med vem som helst på Matrix-nätverket, oavsett om de använder Element eller en annan Matrix-app, och till och med om de använder ett annat meddelandesystem som Slack, IRC eller XMPP.
 
 <b>SUPERSÄKER</b>: Riktig totalsträckskryptering (bara de in konversationen kan avkryptera meddelandena), och korssingering för att verifiera konversationsmedlemmars enheter.
 

matrix-sdk-android-rx/src/main/java/org/matrix/android/sdk/rx/RxRoom.kt

@@ -142,6 +142,10 @@ class RxRoom(private val room: Room) {
     fun updateAvatar(avatarUri: Uri, fileName: String): Completable = completableBuilder<Unit> {
         room.updateAvatar(avatarUri, fileName, it)
     }
+
+    fun deleteAvatar(): Completable = completableBuilder<Unit> {
+        room.deleteAvatar(it)
+    }
 }
 
 fun Room.rx(): RxRoom {

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/common/CommonTestHelper.kt

@@ -88,7 +88,10 @@ class CommonTestHelper(context: Context) {
     fun syncSession(session: Session) {
         val lock = CountDownLatch(1)
 
-        GlobalScope.launch(Dispatchers.Main) { session.open() }
+        val job = GlobalScope.launch(Dispatchers.Main) {
+            session.open()
+        }
+        runBlocking { job.join() }
 
         session.startSync(true)
 
@@ -341,7 +344,7 @@ class CommonTestHelper(context: Context) {
     }
 
     // Transform a method with a MatrixCallback to a synchronous method
-    inline fun <reified T> doSync(block: (MatrixCallback<T>) -> Unit): T {
+    inline fun <reified T> doSync(timeout: Long? = TestConstants.timeOutMillis, block: (MatrixCallback<T>) -> Unit): T {
         val lock = CountDownLatch(1)
         var result: T? = null
 
@@ -354,7 +357,7 @@ class CommonTestHelper(context: Context) {
 
         block.invoke(callback)
 
-        await(lock)
+        await(lock, timeout)
 
         assertNotNull(result)
         return result!!
@@ -366,8 +369,9 @@ class CommonTestHelper(context: Context) {
     fun Iterable<Session>.signOutAndClose() = forEach { signOutAndClose(it) }
 
     fun signOutAndClose(session: Session) {
-        doSync<Unit> { session.signOut(true, it) }
+        doSync<Unit>(60_000) { session.signOut(true, it) }
-        session.close()
+        // no need signout will close
+        // session.close()
     }
 }
 

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/common/CryptoTestHelper.kt

@@ -32,9 +32,6 @@ import org.matrix.android.sdk.api.session.room.model.Membership
 import org.matrix.android.sdk.api.session.room.model.RoomSummary
 import org.matrix.android.sdk.api.session.room.model.create.CreateRoomParams
 import org.matrix.android.sdk.api.session.room.roomSummaryQueryParams
-import org.matrix.android.sdk.api.session.room.timeline.Timeline
-import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
-import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.MegolmBackupAuthData
@@ -197,47 +194,16 @@ class CryptoTestHelper(private val mTestHelper: CommonTestHelper) {
         val roomFromBobPOV = bobSession.getRoom(aliceRoomId)!!
         val roomFromAlicePOV = aliceSession.getRoom(aliceRoomId)!!
 
-        val lock = CountDownLatch(1)
-
-        val bobEventsListener = object : Timeline.Listener {
-            override fun onTimelineFailure(throwable: Throwable) {
-                // noop
-            }
-
-            override fun onNewTimelineEvents(eventIds: List<String>) {
-                // noop
-            }
-
-            override fun onTimelineUpdated(snapshot: List<TimelineEvent>) {
-                val messages = snapshot.filter { it.root.getClearType() == EventType.MESSAGE }
-                        .groupBy { it.root.senderId!! }
-
-                // Alice has sent 2 messages and Bob has sent 3 messages
-                if (messages[aliceSession.myUserId]?.size == 2 && messages[bobSession.myUserId]?.size == 3) {
-                    lock.countDown()
-                }
-            }
-        }
-
-        val bobTimeline = roomFromBobPOV.createTimeline(null, TimelineSettings(20))
-        bobTimeline.start()
-        bobTimeline.addListener(bobEventsListener)
-
         // Alice sends a message
-        roomFromAlicePOV.sendTextMessage(messagesFromAlice[0])
+        mTestHelper.sendTextMessage(roomFromAlicePOV, messagesFromAlice[0], 1)
+//        roomFromAlicePOV.sendTextMessage(messagesFromAlice[0])
 
         // Bob send 3 messages
-        roomFromBobPOV.sendTextMessage(messagesFromBob[0])
+        mTestHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[0], 1)
-        roomFromBobPOV.sendTextMessage(messagesFromBob[1])
+        mTestHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[1], 1)
-        roomFromBobPOV.sendTextMessage(messagesFromBob[2])
+        mTestHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[2], 1)
-
         // Alice sends a message
-        roomFromAlicePOV.sendTextMessage(messagesFromAlice[1])
+        mTestHelper.sendTextMessage(roomFromAlicePOV, messagesFromAlice[1], 1)
-
-        mTestHelper.await(lock)
-
-        bobTimeline.removeListener(bobEventsListener)
-        bobTimeline.dispose()
 
         return cryptoTestData
     }
@@ -279,20 +245,14 @@ class CryptoTestHelper(private val mTestHelper: CommonTestHelper) {
     fun createFakeMegolmBackupCreationInfo(): MegolmBackupCreationInfo {
         return MegolmBackupCreationInfo(
                 algorithm = MXCRYPTO_ALGORITHM_MEGOLM_BACKUP,
-                authData = createFakeMegolmBackupAuthData()
+                authData = createFakeMegolmBackupAuthData(),
+                recoveryKey = "fake"
         )
     }
 
     fun createDM(alice: Session, bob: Session): String {
         val roomId = mTestHelper.doSync<String> {
-            alice.createRoom(
+            alice.createDirectRoom(bob.myUserId, it)
-                    CreateRoomParams().apply {
-                        invitedUserIds.add(bob.myUserId)
-                        setDirectMessage()
-                        enableEncryptionIfInvitedUsersSupportIt = true
-                    },
-                    it
-            )
         }
 
         mTestHelper.waitWithLatch { latch ->

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt

@@ -115,9 +115,8 @@ class KeysBackupTest : InstrumentedTest {
         }
 
         assertEquals(MXCRYPTO_ALGORITHM_MEGOLM_BACKUP, megolmBackupCreationInfo.algorithm)
-        assertNotNull(megolmBackupCreationInfo.authData)
+        assertNotNull(megolmBackupCreationInfo.authData.publicKey)
-        assertNotNull(megolmBackupCreationInfo.authData!!.publicKey)
+        assertNotNull(megolmBackupCreationInfo.authData.signatures)
-        assertNotNull(megolmBackupCreationInfo.authData!!.signatures)
         assertNotNull(megolmBackupCreationInfo.recoveryKey)
 
         stateObserver.stopAndCheckStates(null)
@@ -258,14 +257,14 @@ class KeysBackupTest : InstrumentedTest {
         // - Check encryptGroupSession() returns stg
         val keyBackupData = keysBackup.encryptGroupSession(session)
         assertNotNull(keyBackupData)
-        assertNotNull(keyBackupData.sessionData)
+        assertNotNull(keyBackupData!!.sessionData)
 
         // - Check pkDecryptionFromRecoveryKey() is able to create a OlmPkDecryption
         val decryption = keysBackup.pkDecryptionFromRecoveryKey(keyBackupCreationInfo.recoveryKey)
         assertNotNull(decryption)
         // - Check decryptKeyBackupData() returns stg
         val sessionData = keysBackup
-                .decryptKeyBackupData(keyBackupData,
+                .decryptKeyBackupData(keyBackupData!!,
                         session.olmInboundGroupSession!!.sessionIdentifier(),
                         cryptoTestData.roomId,
                         decryption!!)

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/session/room/send/MarkdownParserTest.kt

@@ -25,6 +25,8 @@ import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.MethodSorters
 import org.matrix.android.sdk.InstrumentedTest
+import org.matrix.android.sdk.internal.session.room.send.pills.MentionLinkSpecComparator
+import org.matrix.android.sdk.internal.session.room.send.pills.TextPillsUtils
 
 /**
  * It will not be possible to test all combinations. For the moment I add a few tests, then, depending on the problem discovered in the wild,
@@ -45,7 +47,8 @@ class MarkdownParserTest : InstrumentedTest {
      */
     private val markdownParser = MarkdownParser(
             Parser.builder().build(),
-            HtmlRenderer.builder().build()
+            HtmlRenderer.builder().softbreak("<br />").build(),
+            TextPillsUtils(MentionLinkSpecComparator())
     )
 
     @Test
@@ -144,12 +147,14 @@ class MarkdownParserTest : InstrumentedTest {
         )
     }
 
+    // TODO. Improve testTypeNewLines function to cover <pre><code class="language-code">test</code></pre>
     @Test
-    fun parseCodeNewLines() {
+    fun parseCodeNewLines_not_passing() {
         testTypeNewLines(
                 name = "code",
-                markdownPattern = "`",
+                markdownPattern = "```",
-                htmlExpectedTag = "code"
+                htmlExpectedTag = "code",
+                softBreak = "\n"
         )
     }
 
@@ -163,7 +168,7 @@ class MarkdownParserTest : InstrumentedTest {
     }
 
     @Test
-    fun parseCode2NewLines() {
+    fun parseCode2NewLines_not_passing() {
         testTypeNewLines(
                 name = "code",
                 markdownPattern = "``",
@@ -181,7 +186,7 @@ class MarkdownParserTest : InstrumentedTest {
     }
 
     @Test
-    fun parseCode3NewLines() {
+    fun parseCode3NewLines_not_passing() {
         testTypeNewLines(
                 name = "code",
                 markdownPattern = "```",
@@ -243,7 +248,7 @@ class MarkdownParserTest : InstrumentedTest {
     }
 
     @Test
-    fun parseBoldNewLines_not_passing() {
+    fun parseBoldNewLines2() {
         "**bold**\nline2".let { markdownParser.parse(it).expect(it, "<strong>bold</strong><br />line2") }
     }
 
@@ -334,13 +339,14 @@ class MarkdownParserTest : InstrumentedTest {
 
     private fun testTypeNewLines(name: String,
                                  markdownPattern: String,
-                                 htmlExpectedTag: String) {
+                                 htmlExpectedTag: String,
+                                 softBreak: String = "<br />") {
         // With new line inside the block
         "$markdownPattern$name\n$name$markdownPattern"
                 .let {
                     markdownParser.parse(it)
                             .expect(expectedText = it,
-                                    expectedFormattedText = "<$htmlExpectedTag>$name<br />$name</$htmlExpectedTag>")
+                                    expectedFormattedText = "<$htmlExpectedTag>$name$softBreak$name</$htmlExpectedTag>")
                 }
 
         // With new line between two blocks
@@ -348,7 +354,7 @@ class MarkdownParserTest : InstrumentedTest {
                 .let {
                     markdownParser.parse(it)
                             .expect(expectedText = it,
-                                    expectedFormattedText = "<$htmlExpectedTag>$name</$htmlExpectedTag><$htmlExpectedTag>$name</$htmlExpectedTag>")
+                                    expectedFormattedText = "<$htmlExpectedTag>$name</$htmlExpectedTag><br /><$htmlExpectedTag>$name</$htmlExpectedTag>")
                 }
     }
 

matrix-sdk-android/src/debug/java/org/matrix/android/sdk/internal/database/RealmDebugTools.kt

@@ -1,68 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.database
-
-import org.matrix.android.sdk.internal.crypto.store.db.model.CrossSigningInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoRoomEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.DeviceInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.GossipingEventEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.IncomingGossipingRequestEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.KeyInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.KeysBackupDataEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.MyDeviceLastSeenInfoEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OutgoingGossipingRequestEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.TrustLevelEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
-import io.realm.Realm
-import io.realm.RealmConfiguration
-import io.realm.kotlin.where
-import timber.log.Timber
-
-object RealmDebugTools {
-    /**
-     * Log info about the crypto DB
-     */
-    fun dumpCryptoDb(realmConfiguration: RealmConfiguration) {
-        Realm.getInstance(realmConfiguration).use {
-            Timber.d("Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}")
-
-            val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) }
-            Timber.d("Realm encryption key : $key")
-
-            // Check if we have data
-            Timber.e("Realm is empty: ${it.isEmpty}")
-
-            Timber.d("Realm has CryptoMetadataEntity: ${it.where<CryptoMetadataEntity>().count()}")
-            Timber.d("Realm has CryptoRoomEntity: ${it.where<CryptoRoomEntity>().count()}")
-            Timber.d("Realm has DeviceInfoEntity: ${it.where<DeviceInfoEntity>().count()}")
-            Timber.d("Realm has KeysBackupDataEntity: ${it.where<KeysBackupDataEntity>().count()}")
-            Timber.d("Realm has OlmInboundGroupSessionEntity: ${it.where<OlmInboundGroupSessionEntity>().count()}")
-            Timber.d("Realm has OlmSessionEntity: ${it.where<OlmSessionEntity>().count()}")
-            Timber.d("Realm has UserEntity: ${it.where<UserEntity>().count()}")
-            Timber.d("Realm has KeyInfoEntity: ${it.where<KeyInfoEntity>().count()}")
-            Timber.d("Realm has CrossSigningInfoEntity: ${it.where<CrossSigningInfoEntity>().count()}")
-            Timber.d("Realm has TrustLevelEntity: ${it.where<TrustLevelEntity>().count()}")
-            Timber.d("Realm has GossipingEventEntity: ${it.where<GossipingEventEntity>().count()}")
-            Timber.d("Realm has IncomingGossipingRequestEntity: ${it.where<IncomingGossipingRequestEntity>().count()}")
-            Timber.d("Realm has OutgoingGossipingRequestEntity: ${it.where<OutgoingGossipingRequestEntity>().count()}")
-            Timber.d("Realm has MyDeviceLastSeenInfoEntity: ${it.where<MyDeviceLastSeenInfoEntity>().count()}")
-        }
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt

@@ -238,4 +238,9 @@ interface Session :
     }
 
     val sharedSecretStorageService: SharedSecretStorageService
+
+    /**
+     * Maintenance API, allows to print outs info on DB size to logcat
+     */
+    fun logDbUsageInfo()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt

@@ -18,6 +18,7 @@ package org.matrix.android.sdk.api.session.crypto
 
 import android.content.Context
 import androidx.lifecycle.LiveData
+import androidx.paging.PagedList
 import org.matrix.android.sdk.api.MatrixCallback
 import org.matrix.android.sdk.api.listeners.ProgressListener
 import org.matrix.android.sdk.api.session.crypto.crosssigning.CrossSigningService
@@ -40,6 +41,7 @@ import org.matrix.android.sdk.internal.crypto.model.event.RoomKeyWithHeldContent
 import org.matrix.android.sdk.internal.crypto.model.rest.DeviceInfo
 import org.matrix.android.sdk.internal.crypto.model.rest.DevicesListResponse
 import org.matrix.android.sdk.internal.crypto.model.rest.RoomKeyRequestBody
+import kotlin.jvm.Throws
 
 interface CryptoService {
 
@@ -142,12 +144,17 @@ interface CryptoService {
     fun removeSessionListener(listener: NewSessionListener)
 
     fun getOutgoingRoomKeyRequests(): List<OutgoingRoomKeyRequest>
+    fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>>
 
     fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
+    fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>>
 
-    fun getGossipingEventsTrail(): List<Event>
+    fun getGossipingEventsTrail(): LiveData<PagedList<Event>>
+    fun getGossipingEvents(): List<Event>
 
     // For testing shared session
     fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int>
     fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent?
+
+    fun logDbUsageInfo()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/RoomService.kt

@@ -35,6 +35,22 @@ interface RoomService {
     fun createRoom(createRoomParams: CreateRoomParams,
                    callback: MatrixCallback<String>): Cancelable
 
+    /**
+     * Create a direct room asynchronously. This is a facility method to create a direct room with the necessary parameters
+     */
+    fun createDirectRoom(otherUserId: String,
+                         callback: MatrixCallback<String>): Cancelable {
+        return createRoom(
+                CreateRoomParams()
+                        .apply {
+                            invitedUserIds.add(otherUserId)
+                            setDirectMessage()
+                            enableEncryptionIfInvitedUsersSupportIt = true
+                        },
+                callback
+        )
+    }
+
     /**
      * Join a room by id
      * @param roomIdOrAlias the roomId or the room alias of the room to join
@@ -113,5 +129,16 @@ interface RoomService {
      */
     fun getChangeMembershipsLive(): LiveData<Map<String, ChangeMembershipState>>
 
-    fun getExistingDirectRoomWithUser(otherUserId: String): Room?
+    /**
+     * Return the roomId of an existing DM with the other user, or null if such room does not exist
+     * A room is a DM if:
+     *  - it is listed in the `m.direct` account data
+     *  - the current user has joined the room
+     *  - the other user is invited or has joined the room
+     *  - it has exactly 2 members
+     * Note:
+     *  - the returning room can be encrypted or not
+     *  - the power level of the users are not taken into account. Normally in a DM, the 2 members are admins of the room
+     */
+    fun getExistingDirectRoomWithUser(otherUserId: String): String?
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/RoomSummary.kt

@@ -63,8 +63,13 @@ data class RoomSummary constructor(
     val hasNewMessages: Boolean
         get() = notificationCount != 0
 
+    val isLowPriority: Boolean
+        get() = hasTag(RoomTag.ROOM_TAG_LOW_PRIORITY)
+
     val isFavorite: Boolean
-        get() = tags.any { it.name == RoomTag.ROOM_TAG_FAVOURITE }
+        get() = hasTag(RoomTag.ROOM_TAG_FAVOURITE)
+
+    fun hasTag(tag: String) = tags.any { it.name == tag }
 
     val canStartCall: Boolean
         get() = joinedMembersCount == 2

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/create/CreateRoomParams.kt

@@ -16,6 +16,7 @@
 
 package org.matrix.android.sdk.api.session.room.model.create
 
+import android.net.Uri
 import org.matrix.android.sdk.api.session.identity.ThreePid
 import org.matrix.android.sdk.api.session.room.model.PowerLevelsContent
 import org.matrix.android.sdk.api.session.room.model.RoomDirectoryVisibility
@@ -51,6 +52,11 @@ class CreateRoomParams {
      */
     var topic: String? = null
 
+    /**
+     * If this is not null, the image uri will be sent to the media server and will be set as a room avatar.
+     */
+    var avatarUri: Uri? = null
+
     /**
      * A list of user IDs to invite to the room.
      * This will tell the server to invite everyone in the list to the newly created room.

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/relation/ReactionInfo.kt

@@ -23,7 +23,7 @@ import com.squareup.moshi.JsonClass
 data class ReactionInfo(
         @Json(name = "rel_type") override val type: String?,
         @Json(name = "event_id") override val eventId: String,
-        val key: String,
+        @Json(name = "key") val key: String,
         // always null for reaction
         @Json(name = "m.in_reply_to") override val inReplyTo: ReplyToContent? = null,
         @Json(name = "option") override val option: Int? = null

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/send/SendService.kt

@@ -123,11 +123,6 @@ interface SendService {
      */
     fun deleteFailedEcho(localEcho: TimelineEvent)
 
-    /**
-     * Delete all the events in one of the sending states
-     */
-    fun clearSendingQueue()
-
     /**
      * Cancel sending a specific event. It has to be in one of the sending states
      */

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/state/StateService.kt

@@ -58,6 +58,11 @@ interface StateService {
      */
     fun updateAvatar(avatarUri: Uri, fileName: String, callback: MatrixCallback<Unit>): Cancelable
 
+    /**
+     * Delete the avatar of the room
+     */
+    fun deleteAvatar(callback: MatrixCallback<Unit>): Cancelable
+
     fun sendStateEvent(eventType: String, stateKey: String?, body: JsonDict, callback: MatrixCallback<Unit>): Cancelable
 
     fun getStateEvent(eventType: String, stateKey: QueryStringValue = QueryStringValue.NoCondition): Event?

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/CryptoSessionInfoProvider.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import com.zhuinden.monarchy.Monarchy
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.internal.database.model.EventEntity
+import org.matrix.android.sdk.internal.database.model.EventEntityFields
+import org.matrix.android.sdk.internal.database.query.whereType
+import org.matrix.android.sdk.internal.di.SessionDatabase
+import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
+import org.matrix.android.sdk.internal.util.fetchCopied
+import javax.inject.Inject
+
+/**
+ * The crypto module needs some information regarding rooms that are stored
+ * in the session DB, this class encapsulate this functionality
+ */
+internal class CryptoSessionInfoProvider @Inject constructor(
+        @SessionDatabase private val monarchy: Monarchy
+) {
+
+    fun isRoomEncrypted(roomId: String): Boolean {
+        val encryptionEvent = monarchy.fetchCopied { realm ->
+            EventEntity.whereType(realm, roomId = roomId, type = EventType.STATE_ROOM_ENCRYPTION)
+                    .contains(EventEntityFields.CONTENT, "\"algorithm\":\"$MXCRYPTO_ALGORITHM_MEGOLM\"")
+                    .isNotNull(EventEntityFields.STATE_KEY) // should be an empty key
+                    .findFirst()
+        }
+        return encryptionEvent != null
+    }
+
+    /**
+     * @param allActive if true return joined as well as invited, if false, only joined
+     */
+     fun getRoomUserIds(roomId: String, allActive: Boolean): List<String> {
+        var userIds: List<String> = emptyList()
+        monarchy.doWithRealm { realm ->
+            userIds = if (allActive) {
+                RoomMemberHelper(realm, roomId).getActiveRoomMemberIds()
+            } else {
+                RoomMemberHelper(realm, roomId).getJoinedRoomMemberIds()
+            }
+        }
+        return userIds
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt

@@ -17,12 +17,10 @@
 package org.matrix.android.sdk.internal.crypto
 
 import android.content.Context
-import android.os.Handler
-import android.os.Looper
 import androidx.annotation.VisibleForTesting
 import androidx.lifecycle.LiveData
+import androidx.paging.PagedList
 import com.squareup.moshi.Types
-import com.zhuinden.monarchy.Monarchy
 import dagger.Lazy
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.CoroutineScope
@@ -51,9 +49,7 @@ import org.matrix.android.sdk.api.session.room.model.Membership
 import org.matrix.android.sdk.api.session.room.model.RoomHistoryVisibility
 import org.matrix.android.sdk.api.session.room.model.RoomHistoryVisibilityContent
 import org.matrix.android.sdk.api.session.room.model.RoomMemberContent
-import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
 import org.matrix.android.sdk.internal.crypto.actions.MegolmSessionDataImporter
-import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
 import org.matrix.android.sdk.internal.crypto.actions.SetDeviceVerificationAction
 import org.matrix.android.sdk.internal.crypto.algorithms.IMXEncrypting
 import org.matrix.android.sdk.internal.crypto.algorithms.IMXWithHeldExtension
@@ -68,7 +64,6 @@ import org.matrix.android.sdk.internal.crypto.model.MXDeviceInfo
 import org.matrix.android.sdk.internal.crypto.model.MXEncryptEventContentResult
 import org.matrix.android.sdk.internal.crypto.model.MXUsersDevicesMap
 import org.matrix.android.sdk.internal.crypto.model.event.EncryptedEventContent
-import org.matrix.android.sdk.internal.crypto.model.event.OlmEventContent
 import org.matrix.android.sdk.internal.crypto.model.event.RoomKeyContent
 import org.matrix.android.sdk.internal.crypto.model.event.RoomKeyWithHeldContent
 import org.matrix.android.sdk.internal.crypto.model.event.SecretSendEventContent
@@ -82,21 +77,15 @@ import org.matrix.android.sdk.internal.crypto.tasks.DeleteDeviceTask
 import org.matrix.android.sdk.internal.crypto.tasks.DeleteDeviceWithUserPasswordTask
 import org.matrix.android.sdk.internal.crypto.tasks.GetDeviceInfoTask
 import org.matrix.android.sdk.internal.crypto.tasks.GetDevicesTask
-import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
 import org.matrix.android.sdk.internal.crypto.tasks.SetDeviceNameTask
 import org.matrix.android.sdk.internal.crypto.tasks.UploadKeysTask
 import org.matrix.android.sdk.internal.crypto.verification.DefaultVerificationService
-import org.matrix.android.sdk.internal.database.model.EventEntity
-import org.matrix.android.sdk.internal.database.model.EventEntityFields
-import org.matrix.android.sdk.internal.database.query.whereType
 import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.MoshiProvider
-import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.extensions.foldToCallback
 import org.matrix.android.sdk.internal.session.SessionScope
 import org.matrix.android.sdk.internal.session.room.membership.LoadRoomMembersTask
-import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
 import org.matrix.android.sdk.internal.session.sync.model.SyncResponse
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.task.TaskThread
@@ -104,11 +93,11 @@ import org.matrix.android.sdk.internal.task.configureWith
 import org.matrix.android.sdk.internal.task.launchToCallback
 import org.matrix.android.sdk.internal.util.JsonCanonicalizer
 import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
-import org.matrix.android.sdk.internal.util.fetchCopied
 import org.matrix.olm.OlmManager
 import timber.log.Timber
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
+import kotlin.jvm.Throws
 import kotlin.math.max
 
 /**
@@ -171,28 +160,16 @@ internal class DefaultCryptoService @Inject constructor(
         private val setDeviceNameTask: SetDeviceNameTask,
         private val uploadKeysTask: UploadKeysTask,
         private val loadRoomMembersTask: LoadRoomMembersTask,
-        @SessionDatabase private val monarchy: Monarchy,
+        private val cryptoSessionInfoProvider: CryptoSessionInfoProvider,
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val taskExecutor: TaskExecutor,
         private val cryptoCoroutineScope: CoroutineScope,
-        private val ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction,
+        private val eventDecryptor: EventDecryptor
-        private val sendToDeviceTask: SendToDeviceTask,
-        private val messageEncrypter: MessageEncrypter
 ) : CryptoService {
 
-    init {
-        verificationService.cryptoService = this
-    }
-
-    private val uiHandler = Handler(Looper.getMainLooper())
-
     private val isStarting = AtomicBoolean(false)
     private val isStarted = AtomicBoolean(false)
 
-    // The date of the last time we forced establishment
-    // of a new session for each user:device.
-    private val lastNewSessionForcedDates = MXUsersDevicesMap<Long>()
-
     fun onStateEvent(roomId: String, event: Event) {
         when (event.getClearType()) {
             EventType.STATE_ROOM_ENCRYPTION         -> onRoomEncryptionEvent(roomId, event)
@@ -209,6 +186,8 @@ internal class DefaultCryptoService @Inject constructor(
         }
     }
 
+    val gossipingBuffer = mutableListOf<Event>()
+
     override fun setDeviceName(deviceId: String, deviceName: String, callback: MatrixCallback<Unit>) {
         setDeviceNameTask
                 .configureWith(SetDeviceNameTask.Params(deviceId, deviceName)) {
@@ -335,6 +314,10 @@ internal class DefaultCryptoService @Inject constructor(
         }
         // Just update
         fetchDevicesList(NoOpMatrixCallback())
+
+        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+            cryptoStore.tidyUpDataBase()
+        }
     }
 
     fun ensureDevice() {
@@ -410,7 +393,7 @@ internal class DefaultCryptoService @Inject constructor(
      */
     fun close() = runBlocking(coroutineDispatchers.crypto) {
         cryptoCoroutineScope.coroutineContext.cancelChildren(CancellationException("Closing crypto module"))
-
+        incomingGossipingRequestManager.close()
         olmDevice.release()
         cryptoStore.close()
     }
@@ -452,6 +435,13 @@ internal class DefaultCryptoService @Inject constructor(
                     incomingGossipingRequestManager.processReceivedGossipingRequests()
                 }
             }
+
+            tryOrNull {
+                gossipingBuffer.toList().let {
+                    cryptoStore.saveGossipingEvents(it)
+                }
+                gossipingBuffer.clear()
+            }
         }
     }
 
@@ -612,13 +602,7 @@ internal class DefaultCryptoService @Inject constructor(
      * @return true if the room is encrypted with algorithm MXCRYPTO_ALGORITHM_MEGOLM
      */
     override fun isRoomEncrypted(roomId: String): Boolean {
-        val encryptionEvent = monarchy.fetchCopied { realm ->
+        return cryptoSessionInfoProvider.isRoomEncrypted(roomId)
-            EventEntity.whereType(realm, roomId = roomId, type = EventType.STATE_ROOM_ENCRYPTION)
-                    .contains(EventEntityFields.CONTENT, "\"algorithm\":\"$MXCRYPTO_ALGORITHM_MEGOLM\"")
-                    .isNotNull(EventEntityFields.STATE_KEY)
-                    .findFirst()
-        }
-        return encryptionEvent != null
     }
 
     /**
@@ -660,11 +644,8 @@ internal class DefaultCryptoService @Inject constructor(
                                      eventType: String,
                                      roomId: String,
                                      callback: MatrixCallback<MXEncryptEventContentResult>) {
+        // moved to crypto scope to have uptodate values
         cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-//            if (!isStarted()) {
-//                Timber.v("## CRYPTO | encryptEventContent() : wait after e2e init")
-//                internalStart(false)
-//            }
             val userIds = getRoomUserIds(roomId)
             var alg = roomEncryptorsStore.get(roomId)
             if (alg == null) {
@@ -720,14 +701,7 @@ internal class DefaultCryptoService @Inject constructor(
      * @param callback the callback to return data or null
      */
     override fun decryptEventAsync(event: Event, timeline: String, callback: MatrixCallback<MXEventDecryptionResult>) {
-        cryptoCoroutineScope.launch {
+        eventDecryptor.decryptEventAsync(event, timeline, callback)
-            val result = runCatching {
-                withContext(coroutineDispatchers.crypto) {
-                    internalDecryptEvent(event, timeline)
-                }
-            }
-            result.foldToCallback(callback)
-        }
     }
 
     /**
@@ -739,42 +713,7 @@ internal class DefaultCryptoService @Inject constructor(
      */
     @Throws(MXCryptoError::class)
     private fun internalDecryptEvent(event: Event, timeline: String): MXEventDecryptionResult {
-        val eventContent = event.content
+       return eventDecryptor.decryptEvent(event, timeline)
-        if (eventContent == null) {
-            Timber.e("## CRYPTO | decryptEvent : empty event content")
-            throw MXCryptoError.Base(MXCryptoError.ErrorType.BAD_ENCRYPTED_MESSAGE, MXCryptoError.BAD_ENCRYPTED_MESSAGE_REASON)
-        } else {
-            val algorithm = eventContent["algorithm"]?.toString()
-            val alg = roomDecryptorProvider.getOrCreateRoomDecryptor(event.roomId, algorithm)
-            if (alg == null) {
-                val reason = String.format(MXCryptoError.UNABLE_TO_DECRYPT_REASON, event.eventId, algorithm)
-                Timber.e("## CRYPTO | decryptEvent() : $reason")
-                throw MXCryptoError.Base(MXCryptoError.ErrorType.UNABLE_TO_DECRYPT, reason)
-            } else {
-                try {
-                    return alg.decryptEvent(event, timeline)
-                } catch (mxCryptoError: MXCryptoError) {
-                    Timber.d("## CRYPTO | internalDecryptEvent : Failed to decrypt ${event.eventId} reason: $mxCryptoError")
-                    if (algorithm == MXCRYPTO_ALGORITHM_OLM) {
-                        if (mxCryptoError is MXCryptoError.Base
-                                && mxCryptoError.errorType == MXCryptoError.ErrorType.BAD_ENCRYPTED_MESSAGE) {
-                            // need to find sending device
-                            val olmContent = event.content.toModel<OlmEventContent>()
-                            cryptoStore.getUserDevices(event.senderId ?: "")
-                                    ?.values
-                                    ?.firstOrNull { it.identityKey() == olmContent?.senderKey }
-                                    ?.let {
-                                        markOlmSessionForUnwedging(event.senderId ?: "", it)
-                                    }
-                                    ?: run {
-                                        Timber.v("## CRYPTO | markOlmSessionForUnwedging() : Failed to find sender crypto device")
-                                    }
-                        }
-                    }
-                    throw mxCryptoError
-                }
-            }
-        }
     }
 
     /**
@@ -796,19 +735,19 @@ internal class DefaultCryptoService @Inject constructor(
         cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
             when (event.getClearType()) {
                 EventType.ROOM_KEY, EventType.FORWARDED_ROOM_KEY -> {
-                    cryptoStore.saveGossipingEvent(event)
+                    gossipingBuffer.add(event)
                     // Keys are imported directly, not waiting for end of sync
                     onRoomKeyEvent(event)
                 }
                 EventType.REQUEST_SECRET,
                 EventType.ROOM_KEY_REQUEST                       -> {
                     // save audit trail
-                    cryptoStore.saveGossipingEvent(event)
+                    gossipingBuffer.add(event)
                     // Requests are stacked, and will be handled one by one at the end of the sync (onSyncComplete)
                     incomingGossipingRequestManager.onGossipingRequestEvent(event)
                 }
                 EventType.SEND_SECRET                            -> {
-                    cryptoStore.saveGossipingEvent(event)
+                    gossipingBuffer.add(event)
                     onSecretSendReceived(event)
                 }
                 EventType.ROOM_KEY_WITHHELD                      -> {
@@ -828,7 +767,7 @@ internal class DefaultCryptoService @Inject constructor(
      */
     private fun onRoomKeyEvent(event: Event) {
         val roomKeyContent = event.getClearContent().toModel<RoomKeyContent>() ?: return
-        Timber.v("## CRYPTO | GOSSIP onRoomKeyEvent() : type<${event.type}> , sessionId<${roomKeyContent.sessionId}>")
+        Timber.v("## CRYPTO | GOSSIP onRoomKeyEvent() : type<${event.getClearType()}> , sessionId<${roomKeyContent.sessionId}>")
         if (roomKeyContent.roomId.isNullOrEmpty() || roomKeyContent.algorithm.isNullOrEmpty()) {
             Timber.e("## CRYPTO | GOSSIP onRoomKeyEvent() : missing fields")
             return
@@ -935,19 +874,9 @@ internal class DefaultCryptoService @Inject constructor(
     }
 
     private fun getRoomUserIds(roomId: String): List<String> {
-        var userIds: List<String> = emptyList()
-        monarchy.doWithRealm { realm ->
-            // Check whether the event content must be encrypted for the invited members.
         val encryptForInvitedMembers = isEncryptionEnabledForInvitedUser()
                 && shouldEncryptForInvitedMembers(roomId)
-
+        return cryptoSessionInfoProvider.getRoomUserIds(roomId, encryptForInvitedMembers)
-            userIds = if (encryptForInvitedMembers) {
-                RoomMemberHelper(realm, roomId).getActiveRoomMemberIds()
-            } else {
-                RoomMemberHelper(realm, roomId).getJoinedRoomMemberIds()
-            }
-        }
-        return userIds
     }
 
     /**
@@ -1257,38 +1186,38 @@ internal class DefaultCryptoService @Inject constructor(
         incomingGossipingRequestManager.removeRoomKeysRequestListener(listener)
     }
 
-    private fun markOlmSessionForUnwedging(senderId: String, deviceInfo: CryptoDeviceInfo) {
+//    private fun markOlmSessionForUnwedging(senderId: String, deviceInfo: CryptoDeviceInfo) {
-        val deviceKey = deviceInfo.identityKey()
+//        val deviceKey = deviceInfo.identityKey()
-
+//
-        val lastForcedDate = lastNewSessionForcedDates.getObject(senderId, deviceKey) ?: 0
+//        val lastForcedDate = lastNewSessionForcedDates.getObject(senderId, deviceKey) ?: 0
-        val now = System.currentTimeMillis()
+//        val now = System.currentTimeMillis()
-        if (now - lastForcedDate < CRYPTO_MIN_FORCE_SESSION_PERIOD_MILLIS) {
+//        if (now - lastForcedDate < CRYPTO_MIN_FORCE_SESSION_PERIOD_MILLIS) {
-            Timber.d("## CRYPTO | markOlmSessionForUnwedging: New session already forced with device at $lastForcedDate. Not forcing another")
+//            Timber.d("## CRYPTO | markOlmSessionForUnwedging: New session already forced with device at $lastForcedDate. Not forcing another")
-            return
+//            return
-        }
+//        }
-
+//
-        Timber.d("## CRYPTO | markOlmSessionForUnwedging from $senderId:${deviceInfo.deviceId}")
+//        Timber.d("## CRYPTO | markOlmSessionForUnwedging from $senderId:${deviceInfo.deviceId}")
-        lastNewSessionForcedDates.setObject(senderId, deviceKey, now)
+//        lastNewSessionForcedDates.setObject(senderId, deviceKey, now)
-
+//
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+//        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-            ensureOlmSessionsForDevicesAction.handle(mapOf(senderId to listOf(deviceInfo)), force = true)
+//            ensureOlmSessionsForDevicesAction.handle(mapOf(senderId to listOf(deviceInfo)), force = true)
-
+//
-            // Now send a blank message on that session so the other side knows about it.
+//            // Now send a blank message on that session so the other side knows about it.
-            // (The keyshare request is sent in the clear so that won't do)
+//            // (The keyshare request is sent in the clear so that won't do)
-            // We send this first such that, as long as the toDevice messages arrive in the
+//            // We send this first such that, as long as the toDevice messages arrive in the
-            // same order we sent them, the other end will get this first, set up the new session,
+//            // same order we sent them, the other end will get this first, set up the new session,
-            // then get the keyshare request and send the key over this new session (because it
+//            // then get the keyshare request and send the key over this new session (because it
-            // is the session it has most recently received a message on).
+//            // is the session it has most recently received a message on).
-            val payloadJson = mapOf<String, Any>("type" to EventType.DUMMY)
+//            val payloadJson = mapOf<String, Any>("type" to EventType.DUMMY)
-
+//
-            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
+//            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
-            val sendToDeviceMap = MXUsersDevicesMap<Any>()
+//            val sendToDeviceMap = MXUsersDevicesMap<Any>()
-            sendToDeviceMap.setObject(senderId, deviceInfo.deviceId, encodedPayload)
+//            sendToDeviceMap.setObject(senderId, deviceInfo.deviceId, encodedPayload)
-            Timber.v("## CRYPTO | markOlmSessionForUnwedging() : sending to $senderId:${deviceInfo.deviceId}")
+//            Timber.v("## CRYPTO | markOlmSessionForUnwedging() : sending to $senderId:${deviceInfo.deviceId}")
-            val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
+//            val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
-            sendToDeviceTask.execute(sendToDeviceParams)
+//            sendToDeviceTask.execute(sendToDeviceParams)
-        }
+//        }
-    }
+//    }
 
     /**
      * Provides the list of unknown devices
@@ -1339,14 +1268,26 @@ internal class DefaultCryptoService @Inject constructor(
         return cryptoStore.getOutgoingRoomKeyRequests()
     }
 
+    override fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>> {
+        return cryptoStore.getOutgoingRoomKeyRequestsPaged()
+    }
+
+    override fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>> {
+        return cryptoStore.getIncomingRoomKeyRequestsPaged()
+    }
+
     override fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest> {
         return cryptoStore.getIncomingRoomKeyRequests()
     }
 
-    override fun getGossipingEventsTrail(): List<Event> {
+    override fun getGossipingEventsTrail(): LiveData<PagedList<Event>> {
         return cryptoStore.getGossipingEventsTrail()
     }
 
+    override fun getGossipingEvents(): List<Event> {
+        return cryptoStore.getGossipingEvents()
+    }
+
     override fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int> {
         return cryptoStore.getSharedWithInfo(roomId, sessionId)
     }
@@ -1354,6 +1295,11 @@ internal class DefaultCryptoService @Inject constructor(
     override fun getWithHeldMegolmSession(roomId: String, sessionId: String): RoomKeyWithHeldContent? {
         return cryptoStore.getWithHeldMegolmSession(roomId, sessionId)
     }
+
+    override fun logDbUsageInfo() {
+        cryptoStore.logDbUsageInfo()
+    }
+
     /* ==========================================================================================
      * For test only
      * ========================================================================================== */

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DeviceListManager.kt

@@ -377,7 +377,7 @@ internal class DeviceListManager @Inject constructor(private val cryptoStore: IM
         }
 
         // Update devices trust for these users
-        dispatchDeviceChange(downloadUsers)
+        // dispatchDeviceChange(downloadUsers)
 
         return onKeysDownloadSucceed(filteredUsers, response.failures)
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/EventDecryptor.kt

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.api.MatrixCallback
+import org.matrix.android.sdk.api.session.crypto.MXCryptoError
+import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
+import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
+import org.matrix.android.sdk.internal.crypto.model.CryptoDeviceInfo
+import org.matrix.android.sdk.internal.crypto.model.MXUsersDevicesMap
+import org.matrix.android.sdk.internal.crypto.model.event.OlmEventContent
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
+import org.matrix.android.sdk.internal.extensions.foldToCallback
+import org.matrix.android.sdk.internal.session.SessionScope
+import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
+import timber.log.Timber
+import javax.inject.Inject
+import kotlin.jvm.Throws
+
+@SessionScope
+internal class EventDecryptor @Inject constructor(
+        private val cryptoCoroutineScope: CoroutineScope,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val roomDecryptorProvider: RoomDecryptorProvider,
+        private val messageEncrypter: MessageEncrypter,
+        private val sendToDeviceTask: SendToDeviceTask,
+        private val ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction,
+        private val cryptoStore: IMXCryptoStore
+) {
+
+    // The date of the last time we forced establishment
+    // of a new session for each user:device.
+    private val lastNewSessionForcedDates = MXUsersDevicesMap<Long>()
+
+    /**
+     * Decrypt an event
+     *
+     * @param event    the raw event.
+     * @param timeline the id of the timeline where the event is decrypted. It is used to prevent replay attack.
+     * @return the MXEventDecryptionResult data, or throw in case of error
+     */
+    @Throws(MXCryptoError::class)
+    fun decryptEvent(event: Event, timeline: String): MXEventDecryptionResult {
+        return internalDecryptEvent(event, timeline)
+    }
+
+    /**
+     * Decrypt an event asynchronously
+     *
+     * @param event    the raw event.
+     * @param timeline the id of the timeline where the event is decrypted. It is used to prevent replay attack.
+     * @param callback the callback to return data or null
+     */
+    fun decryptEventAsync(event: Event, timeline: String, callback: MatrixCallback<MXEventDecryptionResult>) {
+        // is it needed to do that on the crypto scope??
+        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+            runCatching {
+                internalDecryptEvent(event, timeline)
+            }.foldToCallback(callback)
+        }
+    }
+
+    /**
+     * Decrypt an event
+     *
+     * @param event    the raw event.
+     * @param timeline the id of the timeline where the event is decrypted. It is used to prevent replay attack.
+     * @return the MXEventDecryptionResult data, or null in case of error
+     */
+    @Throws(MXCryptoError::class)
+    private fun internalDecryptEvent(event: Event, timeline: String): MXEventDecryptionResult {
+        val eventContent = event.content
+        if (eventContent == null) {
+            Timber.e("## CRYPTO | decryptEvent : empty event content")
+            throw MXCryptoError.Base(MXCryptoError.ErrorType.BAD_ENCRYPTED_MESSAGE, MXCryptoError.BAD_ENCRYPTED_MESSAGE_REASON)
+        } else {
+            val algorithm = eventContent["algorithm"]?.toString()
+            val alg = roomDecryptorProvider.getOrCreateRoomDecryptor(event.roomId, algorithm)
+            if (alg == null) {
+                val reason = String.format(MXCryptoError.UNABLE_TO_DECRYPT_REASON, event.eventId, algorithm)
+                Timber.e("## CRYPTO | decryptEvent() : $reason")
+                throw MXCryptoError.Base(MXCryptoError.ErrorType.UNABLE_TO_DECRYPT, reason)
+            } else {
+                try {
+                    return alg.decryptEvent(event, timeline)
+                } catch (mxCryptoError: MXCryptoError) {
+                    Timber.d("## CRYPTO | internalDecryptEvent : Failed to decrypt ${event.eventId} reason: $mxCryptoError")
+                    if (algorithm == MXCRYPTO_ALGORITHM_OLM) {
+                        if (mxCryptoError is MXCryptoError.Base
+                                && mxCryptoError.errorType == MXCryptoError.ErrorType.BAD_ENCRYPTED_MESSAGE) {
+                            // need to find sending device
+                            cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+                                val olmContent = event.content.toModel<OlmEventContent>()
+                                cryptoStore.getUserDevices(event.senderId ?: "")
+                                        ?.values
+                                        ?.firstOrNull { it.identityKey() == olmContent?.senderKey }
+                                        ?.let {
+                                            markOlmSessionForUnwedging(event.senderId ?: "", it)
+                                        }
+                                        ?: run {
+                                            Timber.v("## CRYPTO | markOlmSessionForUnwedging() : Failed to find sender crypto device")
+                                        }
+                            }
+                        }
+                    }
+                    throw mxCryptoError
+                }
+            }
+        }
+    }
+
+    // coroutineDispatchers.crypto scope
+    private fun markOlmSessionForUnwedging(senderId: String, deviceInfo: CryptoDeviceInfo) {
+        val deviceKey = deviceInfo.identityKey()
+
+        val lastForcedDate = lastNewSessionForcedDates.getObject(senderId, deviceKey) ?: 0
+        val now = System.currentTimeMillis()
+        if (now - lastForcedDate < DefaultCryptoService.CRYPTO_MIN_FORCE_SESSION_PERIOD_MILLIS) {
+            Timber.d("## CRYPTO | markOlmSessionForUnwedging: New session already forced with device at $lastForcedDate. Not forcing another")
+            return
+        }
+
+        Timber.d("## CRYPTO | markOlmSessionForUnwedging from $senderId:${deviceInfo.deviceId}")
+        lastNewSessionForcedDates.setObject(senderId, deviceKey, now)
+
+        // offload this from crypto thread (?)
+        cryptoCoroutineScope.launch(coroutineDispatchers.computation) {
+            ensureOlmSessionsForDevicesAction.handle(mapOf(senderId to listOf(deviceInfo)), force = true)
+
+            // Now send a blank message on that session so the other side knows about it.
+            // (The keyshare request is sent in the clear so that won't do)
+            // We send this first such that, as long as the toDevice messages arrive in the
+            // same order we sent them, the other end will get this first, set up the new session,
+            // then get the keyshare request and send the key over this new session (because it
+            // is the session it has most recently received a message on).
+            val payloadJson = mapOf<String, Any>("type" to EventType.DUMMY)
+
+            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
+            val sendToDeviceMap = MXUsersDevicesMap<Any>()
+            sendToDeviceMap.setObject(senderId, deviceInfo.deviceId, encodedPayload)
+            Timber.v("## CRYPTO | markOlmSessionForUnwedging() : sending to $senderId:${deviceInfo.deviceId}")
+            withContext(coroutineDispatchers.io) {
+                val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
+                sendToDeviceTask.execute(sendToDeviceParams)
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/InboundGroupSessionStore.kt

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import android.util.LruCache
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import org.matrix.android.sdk.api.extensions.tryOrNull
+import org.matrix.android.sdk.internal.crypto.model.OlmInboundGroupSessionWrapper2
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
+import timber.log.Timber
+import java.util.Timer
+import java.util.TimerTask
+import javax.inject.Inject
+
+/**
+ * Allows to cache and batch store operations on inbound group session store.
+ * Because it is used in the decrypt flow, that can be called quite rapidly
+ */
+internal class InboundGroupSessionStore @Inject constructor(
+        private val store: IMXCryptoStore,
+        private val cryptoCoroutineScope: CoroutineScope,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers) {
+
+    private data class CacheKey(
+            val sessionId: String,
+            val senderKey: String
+    )
+
+    private val sessionCache = object : LruCache<CacheKey, OlmInboundGroupSessionWrapper2>(30) {
+        override fun entryRemoved(evicted: Boolean, key: CacheKey?, oldValue: OlmInboundGroupSessionWrapper2?, newValue: OlmInboundGroupSessionWrapper2?) {
+            if (evicted && oldValue != null) {
+                cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+                    Timber.v("## Inbound: entryRemoved ${oldValue.roomId}-${oldValue.senderKey}")
+                    store.storeInboundGroupSessions(listOf(oldValue))
+                }
+            }
+        }
+    }
+
+    private val timer = Timer()
+    private var timerTask: TimerTask? = null
+
+    private val dirtySession = mutableListOf<OlmInboundGroupSessionWrapper2>()
+
+    @Synchronized
+    fun getInboundGroupSession(sessionId: String, senderKey: String): OlmInboundGroupSessionWrapper2? {
+        synchronized(sessionCache) {
+            val known = sessionCache[CacheKey(sessionId, senderKey)]
+            Timber.v("## Inbound: getInboundGroupSession in cache ${known != null}")
+            return known ?: store.getInboundGroupSession(sessionId, senderKey)?.also {
+                Timber.v("## Inbound: getInboundGroupSession cache populate ${it.roomId}")
+                sessionCache.put(CacheKey(sessionId, senderKey), it)
+            }
+        }
+    }
+
+    @Synchronized
+    fun storeInBoundGroupSession(wrapper: OlmInboundGroupSessionWrapper2) {
+        Timber.v("## Inbound: getInboundGroupSession mark as dirty ${wrapper.roomId}-${wrapper.senderKey}")
+        // We want to batch this a bit for performances
+        dirtySession.add(wrapper)
+
+        timerTask?.cancel()
+        timerTask = object : TimerTask() {
+            override fun run() {
+                batchSave()
+            }
+        }
+        timer.schedule(timerTask!!, 2_000)
+    }
+
+    @Synchronized
+    private fun batchSave() {
+        val toSave = mutableListOf<OlmInboundGroupSessionWrapper2>().apply { addAll(dirtySession) }
+        dirtySession.clear()
+        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+            Timber.v("## Inbound: getInboundGroupSession batching save of ${dirtySession.size}")
+            tryOrNull {
+                store.storeInboundGroupSessions(toSave)
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/IncomingGossipingRequestManager.kt

@@ -38,6 +38,7 @@ import org.matrix.android.sdk.internal.session.SessionScope
 import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
 import timber.log.Timber
+import java.util.concurrent.Executors
 import javax.inject.Inject
 
 @SessionScope
@@ -52,6 +53,7 @@ internal class IncomingGossipingRequestManager @Inject constructor(
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val cryptoCoroutineScope: CoroutineScope) {
 
+    private val executor = Executors.newSingleThreadExecutor()
     // list of IncomingRoomKeyRequests/IncomingRoomKeyRequestCancellations
     // we received in the current sync.
     private val receivedGossipingRequests = ArrayList<IncomingShareRequestCommon>()
@@ -64,6 +66,10 @@ internal class IncomingGossipingRequestManager @Inject constructor(
         receivedGossipingRequests.addAll(cryptoStore.getPendingIncomingGossipingRequests())
     }
 
+    fun close() {
+        executor.shutdownNow()
+    }
+
     // Recently verified devices (map of deviceId and timestamp)
     private val recentlyVerifiedDevices = HashMap<String, Long>()
 
@@ -99,7 +105,7 @@ internal class IncomingGossipingRequestManager @Inject constructor(
     fun onGossipingRequestEvent(event: Event) {
         Timber.v("## CRYPTO | GOSSIP onGossipingRequestEvent type ${event.type} from user ${event.senderId}")
         val roomKeyShare = event.getClearContent().toModel<GossipingDefaultContent>()
-        val ageLocalTs = event.unsignedData?.age?.let { System.currentTimeMillis() - it }
+        // val ageLocalTs = event.unsignedData?.age?.let { System.currentTimeMillis() - it }
         when (roomKeyShare?.action) {
             GossipingToDeviceObject.ACTION_SHARE_REQUEST      -> {
                 if (event.getClearType() == EventType.REQUEST_SECRET) {
@@ -108,8 +114,8 @@ internal class IncomingGossipingRequestManager @Inject constructor(
                             // ignore, it was sent by me as *
                             Timber.v("## GOSSIP onGossipingRequestEvent type ${event.type} ignore remote echo")
                         } else {
-                            // save in DB
+//                            // save in DB
-                            cryptoStore.storeIncomingGossipingRequest(it, ageLocalTs)
+//                            cryptoStore.storeIncomingGossipingRequest(it, ageLocalTs)
                             receivedGossipingRequests.add(it)
                         }
                     }
@@ -119,7 +125,7 @@ internal class IncomingGossipingRequestManager @Inject constructor(
                             // ignore, it was sent by me as *
                             Timber.v("## GOSSIP onGossipingRequestEvent type ${event.type} ignore remote echo")
                         } else {
-                            cryptoStore.storeIncomingGossipingRequest(it, ageLocalTs)
+//                            cryptoStore.storeIncomingGossipingRequest(it, ageLocalTs)
                             receivedGossipingRequests.add(it)
                         }
                     }
@@ -144,13 +150,8 @@ internal class IncomingGossipingRequestManager @Inject constructor(
     fun processReceivedGossipingRequests() {
         val roomKeyRequestsToProcess = receivedGossipingRequests.toList()
         receivedGossipingRequests.clear()
-        for (request in roomKeyRequestsToProcess) {
+
-            if (request is IncomingRoomKeyRequest) {
+        Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : ${roomKeyRequestsToProcess.size} request to process")
-                processIncomingRoomKeyRequest(request)
-            } else if (request is IncomingSecretShareRequest) {
-                processIncomingSecretShareRequest(request)
-            }
-        }
 
         var receivedRequestCancellations: List<IncomingRequestCancellation>? = null
 
@@ -161,6 +162,16 @@ internal class IncomingGossipingRequestManager @Inject constructor(
             }
         }
 
+        executor.execute {
+            cryptoStore.storeIncomingGossipingRequests(roomKeyRequestsToProcess)
+            for (request in roomKeyRequestsToProcess) {
+                if (request is IncomingRoomKeyRequest) {
+                    processIncomingRoomKeyRequest(request)
+                } else if (request is IncomingSecretShareRequest) {
+                    processIncomingSecretShareRequest(request)
+                }
+            }
+
             receivedRequestCancellations?.forEach { request ->
                 Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : m.room_key_request cancellation $request")
                 // we should probably only notify the app of cancellations we told it
@@ -183,6 +194,7 @@ internal class IncomingGossipingRequestManager @Inject constructor(
                 }
             }
         }
+    }
 
     private fun processIncomingRoomKeyRequest(request: IncomingRoomKeyRequest) {
         val userId = request.userId ?: return

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/MXOlmDevice.kt

@@ -44,7 +44,9 @@ internal class MXOlmDevice @Inject constructor(
         /**
          * The store where crypto data is saved.
          */
-        private val store: IMXCryptoStore) {
+        private val store: IMXCryptoStore,
+        private val inboundGroupSessionStore: InboundGroupSessionStore
+        ) {
 
     /**
      * @return the Curve25519 key for the account.
@@ -657,7 +659,7 @@ internal class MXOlmDevice @Inject constructor(
                 timelineSet.add(messageIndexKey)
             }
 
-            store.storeInboundGroupSessions(listOf(session))
+            inboundGroupSessionStore.storeInBoundGroupSession(session)
             val payload = try {
                 val adapter = MoshiProvider.providesMoshi().adapter<JsonDict>(JSON_DICT_PARAMETERIZED_TYPE)
                 val payloadString = convertFromUTF8(decryptResult.mDecryptedMessage)
@@ -745,7 +747,7 @@ internal class MXOlmDevice @Inject constructor(
             throw MXCryptoError.Base(MXCryptoError.ErrorType.MISSING_SENDER_KEY, MXCryptoError.ERROR_MISSING_PROPERTY_REASON)
         }
 
-        val session = store.getInboundGroupSession(sessionId, senderKey)
+        val session =  inboundGroupSessionStore.getInboundGroupSession(sessionId, senderKey)
 
         if (session != null) {
             // Check that the room id matches the original one for the session. This stops

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OutgoingGossipingRequestManager.kt

@@ -88,7 +88,7 @@ internal class OutgoingGossipingRequestManager @Inject constructor(
      * @param requestBody requestBody
      */
     fun cancelRoomKeyRequest(requestBody: RoomKeyRequestBody) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.computation) {
             cancelRoomKeyRequest(requestBody, false)
         }
     }
@@ -99,7 +99,7 @@ internal class OutgoingGossipingRequestManager @Inject constructor(
      * @param requestBody requestBody
      */
     fun resendRoomKeyRequest(requestBody: RoomKeyRequestBody) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.computation) {
             cancelRoomKeyRequest(requestBody, true)
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/megolm/MXMegolmEncryption.kt

@@ -16,10 +16,13 @@
 
 package org.matrix.android.sdk.internal.crypto.algorithms.megolm
 
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
 import org.matrix.android.sdk.api.MatrixCallback
 import org.matrix.android.sdk.api.auth.data.Credentials
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
 import org.matrix.android.sdk.api.session.events.model.Content
+import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.internal.crypto.DeviceListManager
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
@@ -39,6 +42,7 @@ import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.task.configureWith
 import org.matrix.android.sdk.internal.util.JsonCanonicalizer
+import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
 import org.matrix.android.sdk.internal.util.convertToUTF8
 import timber.log.Timber
 
@@ -54,7 +58,9 @@ internal class MXMegolmEncryption(
         private val sendToDeviceTask: SendToDeviceTask,
         private val messageEncrypter: MessageEncrypter,
         private val warnOnUnknownDevicesRepository: WarnOnUnknownDeviceRepository,
-        private val taskExecutor: TaskExecutor
+        private val taskExecutor: TaskExecutor,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val cryptoCoroutineScope: CoroutineScope
 ) : IMXEncrypting {
 
     // OutboundSessionInfo. Null if we haven't yet started setting one up. Note
@@ -84,6 +90,8 @@ internal class MXMegolmEncryption(
     }
 
     private fun notifyWithheldForSession(devices: MXUsersDevicesMap<WithHeldCode>, outboundSession: MXOutboundSessionInfo) {
+        // offload to computation thread
+        cryptoCoroutineScope.launch(coroutineDispatchers.computation) {
             mutableListOf<Pair<UserDevice, WithHeldCode>>().apply {
                 devices.forEach { userId, deviceId, withheldCode ->
                     this.add(UserDevice(userId, deviceId) to withheldCode)
@@ -95,6 +103,7 @@ internal class MXMegolmEncryption(
                 notifyKeyWithHeld(targets, outboundSession.sessionId, olmDevice.deviceCurve25519Key, code)
             }
         }
+    }
 
     override fun discardSessionKey() {
         outboundSession = null
@@ -247,6 +256,15 @@ internal class MXMegolmEncryption(
         for ((userId, devicesToShareWith) in devicesByUser) {
             for ((deviceId) in devicesToShareWith) {
                 session.sharedWithHelper.markedSessionAsShared(userId, deviceId, chainIndex)
+                cryptoStore.saveGossipingEvent(Event(
+                        type = EventType.ROOM_KEY,
+                        senderId = credentials.userId,
+                        content = submap.apply {
+                            this["session_key"] = ""
+                            // we add a fake key for trail
+                            this["_dest"] = "$userId|$deviceId"
+                        }
+                ))
             }
         }
 
@@ -420,7 +438,7 @@ internal class MXMegolmEncryption(
             sendToDeviceTask.execute(sendToDeviceParams)
             true
         } catch (failure: Throwable) {
-            Timber.v("## CRYPTO | CRYPTO | reshareKey() : fail to send <$sessionId> to $userId:$deviceId")
+            Timber.e(failure, "## CRYPTO | CRYPTO | reshareKey() : fail to send <$sessionId> to $userId:$deviceId")
             false
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/megolm/MXMegolmEncryptionFactory.kt

@@ -16,6 +16,7 @@
 
 package org.matrix.android.sdk.internal.crypto.algorithms.megolm
 
+import kotlinx.coroutines.CoroutineScope
 import org.matrix.android.sdk.api.auth.data.Credentials
 import org.matrix.android.sdk.internal.crypto.DeviceListManager
 import org.matrix.android.sdk.internal.crypto.MXOlmDevice
@@ -26,6 +27,7 @@ import org.matrix.android.sdk.internal.crypto.repository.WarnOnUnknownDeviceRepo
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
 import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
 import org.matrix.android.sdk.internal.task.TaskExecutor
+import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
 import javax.inject.Inject
 
 internal class MXMegolmEncryptionFactory @Inject constructor(
@@ -38,7 +40,9 @@ internal class MXMegolmEncryptionFactory @Inject constructor(
         private val sendToDeviceTask: SendToDeviceTask,
         private val messageEncrypter: MessageEncrypter,
         private val warnOnUnknownDevicesRepository: WarnOnUnknownDeviceRepository,
-        private val taskExecutor: TaskExecutor) {
+        private val taskExecutor: TaskExecutor,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val cryptoCoroutineScope: CoroutineScope) {
 
     fun create(roomId: String): MXMegolmEncryption {
         return MXMegolmEncryption(
@@ -52,7 +56,9 @@ internal class MXMegolmEncryptionFactory @Inject constructor(
                 sendToDeviceTask,
                 messageEncrypter,
                 warnOnUnknownDevicesRepository,
-                taskExecutor
+                taskExecutor,
+                coroutineDispatchers,
+                cryptoCoroutineScope
         )
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/crosssigning/DefaultCrossSigningService.kt

@@ -17,6 +17,8 @@
 package org.matrix.android.sdk.internal.crypto.crosssigning
 
 import androidx.lifecycle.LiveData
+import androidx.work.BackoffPolicy
+import androidx.work.ExistingWorkPolicy
 import org.matrix.android.sdk.api.MatrixCallback
 import org.matrix.android.sdk.api.extensions.orFalse
 import org.matrix.android.sdk.api.session.crypto.crosssigning.CrossSigningService
@@ -39,15 +41,20 @@ import org.matrix.android.sdk.internal.util.MatrixCoroutineDispatchers
 import org.matrix.android.sdk.internal.util.withoutPrefix
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.launch
-import org.greenrobot.eventbus.EventBus
+import org.matrix.android.sdk.internal.crypto.model.CryptoDeviceInfo
+import org.matrix.android.sdk.internal.di.SessionId
+import org.matrix.android.sdk.internal.di.WorkManagerProvider
+import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
 import org.matrix.olm.OlmPkSigning
 import org.matrix.olm.OlmUtility
 import timber.log.Timber
+import java.util.concurrent.TimeUnit
 import javax.inject.Inject
 
 @SessionScope
 internal class DefaultCrossSigningService @Inject constructor(
         @UserId private val userId: String,
+        @SessionId private val sessionId: String,
         private val cryptoStore: IMXCryptoStore,
         private val deviceListManager: DeviceListManager,
         private val initializeCrossSigningTask: InitializeCrossSigningTask,
@@ -55,7 +62,7 @@ internal class DefaultCrossSigningService @Inject constructor(
         private val taskExecutor: TaskExecutor,
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val cryptoCoroutineScope: CoroutineScope,
-        private val eventBus: EventBus) : CrossSigningService, DeviceListManager.UserDevicesUpdateListener {
+        private val workManagerProvider: WorkManagerProvider) : CrossSigningService, DeviceListManager.UserDevicesUpdateListener {
 
     private var olmUtility: OlmUtility? = null
 
@@ -360,6 +367,12 @@ internal class DefaultCrossSigningService @Inject constructor(
         // First let's get my user key
         val myCrossSigningInfo = cryptoStore.getCrossSigningInfo(userId)
 
+        checkOtherMSKTrusted(myCrossSigningInfo, cryptoStore.getCrossSigningInfo(otherUserId))
+
+        return UserTrustResult.Success
+    }
+
+    fun checkOtherMSKTrusted(myCrossSigningInfo: MXCrossSigningInfo?, otherInfo: MXCrossSigningInfo?): UserTrustResult {
         val myUserKey = myCrossSigningInfo?.userKey()
                 ?: return UserTrustResult.CrossSigningNotConfigured(userId)
 
@@ -368,15 +381,15 @@ internal class DefaultCrossSigningService @Inject constructor(
         }
 
         // Let's get the other user  master key
-        val otherMasterKey = cryptoStore.getCrossSigningInfo(otherUserId)?.masterKey()
+        val otherMasterKey = otherInfo?.masterKey()
-                ?: return UserTrustResult.UnknownCrossSignatureInfo(otherUserId)
+                ?: return UserTrustResult.UnknownCrossSignatureInfo(otherInfo?.userId ?: "")
 
         val masterKeySignaturesMadeByMyUserKey = otherMasterKey.signatures
                 ?.get(userId) // Signatures made by me
                 ?.get("ed25519:${myUserKey.unpaddedBase64PublicKey}")
 
         if (masterKeySignaturesMadeByMyUserKey.isNullOrBlank()) {
-            Timber.d("## CrossSigning  checkUserTrust false for $otherUserId, not signed by my UserSigningKey")
+            Timber.d("## CrossSigning  checkUserTrust false for ${otherInfo.userId}, not signed by my UserSigningKey")
             return UserTrustResult.KeyNotSigned(otherMasterKey)
         }
 
@@ -396,6 +409,15 @@ internal class DefaultCrossSigningService @Inject constructor(
         // and that MSK is trusted (i know the private key, or is signed by a trusted device)
         val myCrossSigningInfo = cryptoStore.getCrossSigningInfo(userId)
 
+        return checkSelfTrust(myCrossSigningInfo, cryptoStore.getUserDeviceList(userId))
+    }
+
+    fun checkSelfTrust(myCrossSigningInfo: MXCrossSigningInfo?, myDevices: List<CryptoDeviceInfo>?): UserTrustResult {
+        // Special case when it's me,
+        // I have to check that MSK -> USK -> SSK
+        // and that MSK is trusted (i know the private key, or is signed by a trusted device)
+//        val myCrossSigningInfo = cryptoStore.getCrossSigningInfo(userId)
+
         val myMasterKey = myCrossSigningInfo?.masterKey()
                 ?: return UserTrustResult.CrossSigningNotConfigured(userId)
 
@@ -423,7 +445,7 @@ internal class DefaultCrossSigningService @Inject constructor(
             // Maybe it's signed by a locally trusted device?
             myMasterKey.signatures?.get(userId)?.forEach { (key, value) ->
                 val potentialDeviceId = key.withoutPrefix("ed25519:")
-                val potentialDevice = cryptoStore.getUserDevice(userId, potentialDeviceId)
+                val potentialDevice = myDevices?.firstOrNull { it.deviceId == potentialDeviceId } // cryptoStore.getUserDevice(userId, potentialDeviceId)
                 if (potentialDevice != null && potentialDevice.isVerified) {
                     // Check signature validity?
                     try {
@@ -561,6 +583,8 @@ internal class DefaultCrossSigningService @Inject constructor(
         cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
             cryptoStore.markMyMasterKeyAsLocallyTrusted(true)
             checkSelfTrust()
+            // re-verify all trusts
+            onUsersDeviceUpdate(listOf(userId))
         }
     }
 
@@ -666,6 +690,55 @@ internal class DefaultCrossSigningService @Inject constructor(
         return DeviceTrustResult.Success(DeviceTrustLevel(crossSigningVerified = true, locallyVerified = locallyTrusted))
     }
 
+    fun checkDeviceTrust(myKeys: MXCrossSigningInfo?, otherKeys: MXCrossSigningInfo?, otherDevice: CryptoDeviceInfo) : DeviceTrustResult {
+        val locallyTrusted = otherDevice.trustLevel?.isLocallyVerified()
+        myKeys ?: return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.CrossSigningNotConfigured(userId))
+
+        if (!myKeys.isTrusted()) return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.KeysNotTrusted(myKeys))
+
+        otherKeys ?: return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.CrossSigningNotConfigured(otherDevice.userId))
+
+        // TODO should we force verification ?
+        if (!otherKeys.isTrusted()) return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.KeysNotTrusted(otherKeys))
+
+        // Check if the trust chain is valid
+        /*
+         *  ┏━━━━━━━━┓                             ┏━━━━━━━━┓
+         *  ┃ ALICE  ┃                             ┃  BOB   ┃
+         *  ┗━━━━━━━━┛                             ┗━━━━━━━━┛
+         *   MSK                      ┌────────────▶MSK
+         *                            │
+         *     │                      │               │
+         *     │    SSK               │               └──▶ SSK  ──────────────────┐
+         *     │                      │                                           │
+         *     │                      │                    USK                    │
+         *     └──▶ USK   ────────────┘              (not visible by              │
+         *                                                Alice)                  │
+         *                                                                        ▼
+         *                                                                ┌──────────────┐
+         *                                                                │ BOB's Device │
+         *                                                                └──────────────┘
+         */
+
+        val otherSSKSignature = otherDevice.signatures?.get(otherKeys.userId)?.get("ed25519:${otherKeys.selfSigningKey()?.unpaddedBase64PublicKey}")
+                ?: return legacyFallbackTrust(
+                        locallyTrusted,
+                        DeviceTrustResult.MissingDeviceSignature(otherDevice.deviceId, otherKeys.selfSigningKey()
+                                ?.unpaddedBase64PublicKey
+                                ?: ""
+                        )
+                )
+
+        // Check  bob's device is signed by bob's SSK
+        try {
+            olmUtility!!.verifyEd25519Signature(otherSSKSignature, otherKeys.selfSigningKey()?.unpaddedBase64PublicKey, otherDevice.canonicalSignable())
+        } catch (e: Throwable) {
+            return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.InvalidDeviceSignature(otherDevice.deviceId, otherSSKSignature, e))
+        }
+
+        return DeviceTrustResult.Success(DeviceTrustLevel(crossSigningVerified = true, locallyVerified = locallyTrusted))
+    }
+
     private fun legacyFallbackTrust(locallyTrusted: Boolean?, crossSignTrustFail: DeviceTrustResult): DeviceTrustResult {
         return if (locallyTrusted == true) {
             DeviceTrustResult.Success(DeviceTrustLevel(crossSigningVerified = false, locallyVerified = true))
@@ -675,36 +748,18 @@ internal class DefaultCrossSigningService @Inject constructor(
     }
 
     override fun onUsersDeviceUpdate(userIds: List<String>) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+        Timber.d("## CrossSigning - onUsersDeviceUpdate for $userIds")
-            Timber.d("## CrossSigning - onUsersDeviceUpdate for ${userIds.size} users")
+        val workerParams = UpdateTrustWorker.Params(sessionId = sessionId, updatedUserIds = userIds)
-            userIds.forEach { otherUserId ->
+        val workerData = WorkerParamsFactory.toData(workerParams)
-                checkUserTrust(otherUserId).let {
-                    Timber.v("## CrossSigning - update trust for $otherUserId , verified=${it.isVerified()}")
-                    setUserKeysAsTrusted(otherUserId, it.isVerified())
-                }
-            }
-        }
 
-        // now check device trust
+        val workRequest = workManagerProvider.matrixOneTimeWorkRequestBuilder<UpdateTrustWorker>()
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+                .setInputData(workerData)
-            userIds.forEach { otherUserId ->
+                .setBackoffCriteria(BackoffPolicy.LINEAR, 2_000L, TimeUnit.MILLISECONDS)
-                // TODO if my keys have changes, i should recheck all devices of all users?
+                .build()
-                val devices = cryptoStore.getUserDeviceList(otherUserId)
-                devices?.forEach { device ->
-                    val updatedTrust = checkDeviceTrust(otherUserId, device.deviceId, device.trustLevel?.isLocallyVerified() ?: false)
-                    Timber.v("## CrossSigning - update trust for device ${device.deviceId} of user $otherUserId , verified=$updatedTrust")
-                    cryptoStore.setDeviceTrust(otherUserId, device.deviceId, updatedTrust.isCrossSignedVerified(), updatedTrust.isLocallyVerified())
-                }
 
-                if (otherUserId == userId) {
+        workManagerProvider.workManager
-                    // It's me, i should check if a newly trusted device is signing my master key
+                .beginUniqueWork("TRUST_UPDATE_QUEUE", ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest)
-                    // In this case it will change my MSK trust, and should then re-trigger a check of all other user trust
+                .enqueue()
-                    setUserKeysAsTrusted(otherUserId, checkSelfTrust().isVerified())
-                }
-            }
-
-            eventBus.post(CryptoToSessionUserTrustChange(userIds))
-        }
     }
 
     private fun setUserKeysAsTrusted(otherUserId: String, trusted: Boolean) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/crosssigning/ShieldTrustUpdater.kt

@@ -1,126 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.matrix.android.sdk.internal.crypto.crosssigning
-
-import org.matrix.android.sdk.api.extensions.orFalse
-import org.matrix.android.sdk.internal.database.model.RoomMemberSummaryEntity
-import org.matrix.android.sdk.internal.database.model.RoomMemberSummaryEntityFields
-import org.matrix.android.sdk.internal.database.model.RoomSummaryEntity
-import org.matrix.android.sdk.internal.database.query.where
-import org.matrix.android.sdk.internal.di.SessionDatabase
-import org.matrix.android.sdk.internal.session.SessionLifecycleObserver
-import org.matrix.android.sdk.internal.session.room.summary.RoomSummaryUpdater
-import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
-import org.matrix.android.sdk.internal.task.TaskExecutor
-import org.matrix.android.sdk.internal.util.createBackgroundHandler
-import io.realm.Realm
-import io.realm.RealmConfiguration
-import kotlinx.coroutines.android.asCoroutineDispatcher
-import kotlinx.coroutines.launch
-import org.greenrobot.eventbus.EventBus
-import org.greenrobot.eventbus.Subscribe
-import timber.log.Timber
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.atomic.AtomicReference
-import javax.inject.Inject
-
-internal class ShieldTrustUpdater @Inject constructor(
-        private val eventBus: EventBus,
-        private val computeTrustTask: ComputeTrustTask,
-        private val taskExecutor: TaskExecutor,
-        @SessionDatabase private val sessionRealmConfiguration: RealmConfiguration,
-        private val roomSummaryUpdater: RoomSummaryUpdater
-) : SessionLifecycleObserver {
-
-    companion object {
-        private val BACKGROUND_HANDLER = createBackgroundHandler("SHIELD_CRYPTO_DB_THREAD")
-        private val BACKGROUND_HANDLER_DISPATCHER = BACKGROUND_HANDLER.asCoroutineDispatcher()
-    }
-
-    private val backgroundSessionRealm = AtomicReference<Realm>()
-
-    private val isStarted = AtomicBoolean()
-
-    override fun onStart() {
-        if (isStarted.compareAndSet(false, true)) {
-            eventBus.register(this)
-            BACKGROUND_HANDLER.post {
-                backgroundSessionRealm.set(Realm.getInstance(sessionRealmConfiguration))
-            }
-        }
-    }
-
-    override fun onStop() {
-        if (isStarted.compareAndSet(true, false)) {
-            eventBus.unregister(this)
-            BACKGROUND_HANDLER.post {
-                backgroundSessionRealm.getAndSet(null).also {
-                    it?.close()
-                }
-            }
-        }
-    }
-
-    @Subscribe
-    fun onRoomMemberChange(update: SessionToCryptoRoomMembersUpdate) {
-        if (!isStarted.get()) {
-            return
-        }
-        taskExecutor.executorScope.launch(BACKGROUND_HANDLER_DISPATCHER) {
-            val updatedTrust = computeTrustTask.execute(ComputeTrustTask.Params(update.userIds, update.isDirect))
-            // We need to send that back to session base
-            backgroundSessionRealm.get()?.executeTransaction { realm ->
-                roomSummaryUpdater.updateShieldTrust(realm, update.roomId, updatedTrust)
-            }
-        }
-    }
-
-    @Subscribe
-    fun onTrustUpdate(update: CryptoToSessionUserTrustChange) {
-        if (!isStarted.get()) {
-            return
-        }
-        onCryptoDevicesChange(update.userIds)
-    }
-
-    private fun onCryptoDevicesChange(users: List<String>) {
-        taskExecutor.executorScope.launch(BACKGROUND_HANDLER_DISPATCHER) {
-            val realm = backgroundSessionRealm.get() ?: return@launch
-            val distinctRoomIds = realm.where(RoomMemberSummaryEntity::class.java)
-                    .`in`(RoomMemberSummaryEntityFields.USER_ID, users.toTypedArray())
-                    .distinct(RoomMemberSummaryEntityFields.ROOM_ID)
-                    .findAll()
-                    .map { it.roomId }
-
-            distinctRoomIds.forEach { roomId ->
-                val roomSummary = RoomSummaryEntity.where(realm, roomId).findFirst()
-                if (roomSummary?.isEncrypted.orFalse()) {
-                    val allActiveRoomMembers = RoomMemberHelper(realm, roomId).getActiveRoomMemberIds()
-                    try {
-                        val updatedTrust = computeTrustTask.execute(
-                                ComputeTrustTask.Params(allActiveRoomMembers, roomSummary?.isDirect == true)
-                        )
-                        realm.executeTransaction {
-                            roomSummaryUpdater.updateShieldTrust(it, roomId, updatedTrust)
-                        }
-                    } catch (failure: Throwable) {
-                        Timber.e(failure)
-                    }
-                }
-            }
-        }
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/crosssigning/UpdateTrustWorker.kt

@@ -0,0 +1,322 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto.crosssigning
+
+import android.content.Context
+import androidx.work.WorkerParameters
+import com.squareup.moshi.JsonClass
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import io.realm.kotlin.where
+import org.matrix.android.sdk.api.crypto.RoomEncryptionTrustLevel
+import org.matrix.android.sdk.api.extensions.orFalse
+import org.matrix.android.sdk.api.session.crypto.crosssigning.MXCrossSigningInfo
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.crypto.store.db.mapper.CrossSigningKeysMapper
+import org.matrix.android.sdk.internal.crypto.store.db.model.CrossSigningInfoEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.CrossSigningInfoEntityFields
+import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMapper
+import org.matrix.android.sdk.internal.crypto.store.db.model.TrustLevelEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields
+import org.matrix.android.sdk.internal.database.model.RoomMemberSummaryEntity
+import org.matrix.android.sdk.internal.database.model.RoomMemberSummaryEntityFields
+import org.matrix.android.sdk.internal.database.model.RoomSummaryEntity
+import org.matrix.android.sdk.internal.database.query.where
+import org.matrix.android.sdk.internal.di.CryptoDatabase
+import org.matrix.android.sdk.internal.di.SessionDatabase
+import org.matrix.android.sdk.internal.di.UserId
+import org.matrix.android.sdk.internal.session.SessionComponent
+import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
+import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
+import org.matrix.android.sdk.internal.worker.SessionWorkerParams
+import timber.log.Timber
+import javax.inject.Inject
+
+internal class UpdateTrustWorker(context: Context,
+                                 params: WorkerParameters)
+    : SessionSafeCoroutineWorker<UpdateTrustWorker.Params>(context, params, Params::class.java) {
+
+    @JsonClass(generateAdapter = true)
+    internal data class Params(
+            override val sessionId: String,
+            override val lastFailureMessage: String? = null,
+            val updatedUserIds: List<String>
+    ) : SessionWorkerParams
+
+    @Inject lateinit var crossSigningService: DefaultCrossSigningService
+
+    // It breaks the crypto store contract, but we need to batch things :/
+    @CryptoDatabase @Inject lateinit var realmConfiguration: RealmConfiguration
+    @UserId @Inject lateinit var myUserId: String
+    @Inject lateinit var crossSigningKeysMapper: CrossSigningKeysMapper
+    @SessionDatabase @Inject lateinit var sessionRealmConfiguration: RealmConfiguration
+
+    //    @Inject lateinit var roomSummaryUpdater: RoomSummaryUpdater
+    @Inject lateinit var cryptoStore: IMXCryptoStore
+
+    override fun injectWith(injector: SessionComponent) {
+        injector.inject(this)
+    }
+
+    override suspend fun doSafeWork(params: Params): Result {
+        var userList = params.updatedUserIds
+        // Unfortunately we don't have much info on what did exactly changed (is it the cross signing keys of that user,
+        // or a new device?) So we check all again :/
+
+        Timber.d("## CrossSigning - Updating trust for $userList")
+
+        // First we check that the users MSK are trusted by mine
+        // After that we check the trust chain for each devices of each users
+        Realm.getInstance(realmConfiguration).use { realm ->
+            realm.executeTransaction {
+                // By mapping here to model, this object is not live
+                // I should update it if needed
+                var myCrossSigningInfo = realm.where(CrossSigningInfoEntity::class.java)
+                        .equalTo(CrossSigningInfoEntityFields.USER_ID, myUserId)
+                        .findFirst()?.let { mapCrossSigningInfoEntity(it) }
+
+                var myTrustResult: UserTrustResult? = null
+
+                if (userList.contains(myUserId)) {
+                    Timber.d("## CrossSigning - Clear all trust as a change on my user was detected")
+                    // i am in the list.. but i don't know exactly the delta of change :/
+                    // If it's my cross signing keys we should refresh all trust
+                    // do it anyway ?
+                    userList = realm.where(CrossSigningInfoEntity::class.java)
+                            .findAll().mapNotNull { it.userId }
+                    Timber.d("## CrossSigning - Updating trust for all $userList")
+
+                    // check right now my keys and mark it as trusted as other trust depends on it
+                    val myDevices = realm.where<UserEntity>()
+                            .equalTo(UserEntityFields.USER_ID, myUserId)
+                            .findFirst()
+                            ?.devices
+                            ?.map { deviceInfo ->
+                                CryptoMapper.mapToModel(deviceInfo)
+                            }
+                    myTrustResult = crossSigningService.checkSelfTrust(myCrossSigningInfo, myDevices).also {
+                        updateCrossSigningKeysTrust(realm, myUserId, it.isVerified())
+                        // update model reference
+                        myCrossSigningInfo = realm.where(CrossSigningInfoEntity::class.java)
+                                .equalTo(CrossSigningInfoEntityFields.USER_ID, myUserId)
+                                .findFirst()?.let { mapCrossSigningInfoEntity(it) }
+                    }
+                }
+
+                val otherInfos = userList.map {
+                    it to realm.where(CrossSigningInfoEntity::class.java)
+                            .equalTo(CrossSigningInfoEntityFields.USER_ID, it)
+                            .findFirst()?.let { mapCrossSigningInfoEntity(it) }
+                }
+                        .toMap()
+
+                val trusts = otherInfos.map { infoEntry ->
+                    infoEntry.key to when (infoEntry.key) {
+                        myUserId -> myTrustResult
+                        else     -> {
+                            crossSigningService.checkOtherMSKTrusted(myCrossSigningInfo, infoEntry.value).also {
+                                Timber.d("## CrossSigning - user:${infoEntry.key} result:$it")
+                            }
+                        }
+                    }
+                }.toMap()
+
+                // TODO! if it's me and my keys has changed... I have to reset trust for everyone!
+                // i have all the new trusts, update DB
+                trusts.forEach {
+                    val verified = it.value?.isVerified() == true
+                    updateCrossSigningKeysTrust(realm, it.key, verified)
+                }
+
+                // Ok so now we have to check device trust for all these users..
+                Timber.v("## CrossSigning - Updating devices cross trust users ${trusts.keys}")
+                trusts.keys.forEach {
+                    val devicesEntities = realm.where<UserEntity>()
+                            .equalTo(UserEntityFields.USER_ID, it)
+                            .findFirst()
+                            ?.devices
+
+                    val trustMap = devicesEntities?.map { device ->
+                        // get up to date from DB has could have been updated
+                        val otherInfo = realm.where(CrossSigningInfoEntity::class.java)
+                                .equalTo(CrossSigningInfoEntityFields.USER_ID, it)
+                                .findFirst()?.let { mapCrossSigningInfoEntity(it) }
+                        device to crossSigningService.checkDeviceTrust(myCrossSigningInfo, otherInfo, CryptoMapper.mapToModel(device))
+                    }?.toMap()
+
+                    // Update trust if needed
+                    devicesEntities?.forEach { device ->
+                        val crossSignedVerified = trustMap?.get(device)?.isCrossSignedVerified()
+                        Timber.d("## CrossSigning - Trust for ${device.userId}|${device.deviceId} : cross verified: ${trustMap?.get(device)}")
+                        if (device.trustLevelEntity?.crossSignedVerified != crossSignedVerified) {
+                            Timber.d("## CrossSigning - Trust change detected for ${device.userId}|${device.deviceId} : cross verified: $crossSignedVerified")
+                            // need to save
+                            val trustEntity = device.trustLevelEntity
+                            if (trustEntity == null) {
+                                realm.createObject(TrustLevelEntity::class.java).let {
+                                    it.locallyVerified = false
+                                    it.crossSignedVerified = crossSignedVerified
+                                    device.trustLevelEntity = it
+                                }
+                            } else {
+                                trustEntity.crossSignedVerified = crossSignedVerified
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // So Cross Signing keys trust is updated, device trust is updated
+        // We can now update room shields? in the session DB?
+
+        Timber.d("## CrossSigning - Updating shields for impacted rooms...")
+        Realm.getInstance(sessionRealmConfiguration).use { it ->
+            it.executeTransaction { realm ->
+                val distinctRoomIds = realm.where(RoomMemberSummaryEntity::class.java)
+                        .`in`(RoomMemberSummaryEntityFields.USER_ID, userList.toTypedArray())
+                        .distinct(RoomMemberSummaryEntityFields.ROOM_ID)
+                        .findAll()
+                        .map { it.roomId }
+                Timber.d("## CrossSigning -  ... impacted rooms $distinctRoomIds")
+                distinctRoomIds.forEach { roomId ->
+                    val roomSummary = RoomSummaryEntity.where(realm, roomId).findFirst()
+                    if (roomSummary?.isEncrypted == true) {
+                        Timber.d("## CrossSigning - Check shield state for room $roomId")
+                        val allActiveRoomMembers = RoomMemberHelper(realm, roomId).getActiveRoomMemberIds()
+                        try {
+                            val updatedTrust = computeRoomShield(allActiveRoomMembers, roomSummary)
+                            if (roomSummary.roomEncryptionTrustLevel != updatedTrust) {
+                                Timber.d("## CrossSigning - Shield change detected for $roomId -> $updatedTrust")
+                                roomSummary.roomEncryptionTrustLevel = updatedTrust
+                            }
+                        } catch (failure: Throwable) {
+                            Timber.e(failure)
+                        }
+                    }
+                }
+            }
+        }
+
+        return Result.success()
+    }
+
+    private fun updateCrossSigningKeysTrust(realm: Realm, userId: String, verified: Boolean) {
+        val xInfoEntity = realm.where(CrossSigningInfoEntity::class.java)
+                .equalTo(CrossSigningInfoEntityFields.USER_ID, userId)
+                .findFirst()
+        xInfoEntity?.crossSigningKeys?.forEach { info ->
+            // optimization to avoid trigger updates when there is no change..
+            if (info.trustLevelEntity?.isVerified() != verified) {
+                Timber.d("## CrossSigning - Trust change for $userId : $verified")
+                val level = info.trustLevelEntity
+                if (level == null) {
+                    val newLevel = realm.createObject(TrustLevelEntity::class.java)
+                    newLevel.locallyVerified = verified
+                    newLevel.crossSignedVerified = verified
+                    info.trustLevelEntity = newLevel
+                } else {
+                    level.locallyVerified = verified
+                    level.crossSignedVerified = verified
+                }
+            }
+        }
+    }
+
+    private fun computeRoomShield(activeMemberUserIds: List<String>, roomSummaryEntity: RoomSummaryEntity): RoomEncryptionTrustLevel {
+        Timber.d("## CrossSigning - computeRoomShield ${roomSummaryEntity.roomId} -> $activeMemberUserIds")
+        // The set of “all users” depends on the type of room:
+        // For regular / topic rooms, all users including yourself, are considered when decorating a room
+        // For 1:1 and group DM rooms, all other users (i.e. excluding yourself) are considered when decorating a room
+        val listToCheck = if (roomSummaryEntity.isDirect) {
+            activeMemberUserIds.filter { it != myUserId }
+        } else {
+            activeMemberUserIds
+        }
+
+        val allTrustedUserIds = listToCheck
+                .filter { userId ->
+                    Realm.getInstance(realmConfiguration).use {
+                        it.where(CrossSigningInfoEntity::class.java)
+                                .equalTo(CrossSigningInfoEntityFields.USER_ID, userId)
+                                .findFirst()?.let { mapCrossSigningInfoEntity(it) }?.isTrusted() == true
+                    }
+                }
+        val myCrossKeys = Realm.getInstance(realmConfiguration).use {
+            it.where(CrossSigningInfoEntity::class.java)
+                    .equalTo(CrossSigningInfoEntityFields.USER_ID, myUserId)
+                    .findFirst()?.let { mapCrossSigningInfoEntity(it) }
+        }
+
+        return if (allTrustedUserIds.isEmpty()) {
+            RoomEncryptionTrustLevel.Default
+        } else {
+            // If one of the verified user as an untrusted device -> warning
+            // If all devices of all verified users are trusted -> green
+            // else -> black
+            allTrustedUserIds
+                    .mapNotNull { uid ->
+                        Realm.getInstance(realmConfiguration).use {
+                            it.where<UserEntity>()
+                                    .equalTo(UserEntityFields.USER_ID, uid)
+                                    .findFirst()
+                                    ?.devices
+                                    ?.map {
+                                        CryptoMapper.mapToModel(it)
+                                    }
+                        }
+                    }
+                    .flatten()
+                    .let { allDevices ->
+                        Timber.v("## CrossSigning - computeRoomShield ${roomSummaryEntity.roomId} devices ${allDevices.map { it.deviceId }}")
+                        if (myCrossKeys != null) {
+                            allDevices.any { !it.trustLevel?.crossSigningVerified.orFalse() }
+                        } else {
+                            // Legacy method
+                            allDevices.any { !it.isVerified }
+                        }
+                    }
+                    .let { hasWarning ->
+                        if (hasWarning) {
+                            RoomEncryptionTrustLevel.Warning
+                        } else {
+                            if (listToCheck.size == allTrustedUserIds.size) {
+                                // all users are trusted and all devices are verified
+                                RoomEncryptionTrustLevel.Trusted
+                            } else {
+                                RoomEncryptionTrustLevel.Default
+                            }
+                        }
+                    }
+        }
+    }
+
+    private fun mapCrossSigningInfoEntity(xsignInfo: CrossSigningInfoEntity): MXCrossSigningInfo {
+        val userId = xsignInfo.userId ?: ""
+        return MXCrossSigningInfo(
+                userId = userId,
+                crossSigningKeys = xsignInfo.crossSigningKeys.mapNotNull {
+                    crossSigningKeysMapper.map(userId, it)
+                }
+        )
+    }
+
+    override fun buildErrorParams(params: Params, message: String): Params {
+        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt

@@ -30,7 +30,6 @@ import org.matrix.android.sdk.api.listeners.StepProgressListener
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupService
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupState
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupStateListener
-import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import org.matrix.android.sdk.internal.crypto.MXOlmDevice
 import org.matrix.android.sdk.internal.crypto.MegolmSessionData
@@ -85,6 +84,7 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.internal.crypto.keysbackup.model.SignalableMegolmBackupAuthData
 import org.matrix.olm.OlmException
 import org.matrix.olm.OlmPkDecryption
 import org.matrix.olm.OlmPkEncryption
@@ -170,7 +170,7 @@ internal class DefaultKeysBackupService @Inject constructor(
             runCatching {
                 withContext(coroutineDispatchers.crypto) {
                     val olmPkDecryption = OlmPkDecryption()
-                    val megolmBackupAuthData = if (password != null) {
+                    val signalableMegolmBackupAuthData = if (password != null) {
                         // Generate a private key from the password
                         val backgroundProgressListener = if (progressListener == null) {
                             null
@@ -189,7 +189,7 @@ internal class DefaultKeysBackupService @Inject constructor(
                         }
 
                         val generatePrivateKeyResult = generatePrivateKeyWithPassword(password, backgroundProgressListener)
-                        MegolmBackupAuthData(
+                        SignalableMegolmBackupAuthData(
                                 publicKey = olmPkDecryption.setPrivateKey(generatePrivateKeyResult.privateKey),
                                 privateKeySalt = generatePrivateKeyResult.salt,
                                 privateKeyIterations = generatePrivateKeyResult.iterations
@@ -197,14 +197,17 @@ internal class DefaultKeysBackupService @Inject constructor(
                     } else {
                         val publicKey = olmPkDecryption.generateKey()
 
-                        MegolmBackupAuthData(
+                        SignalableMegolmBackupAuthData(
                                 publicKey = publicKey
                         )
                     }
 
-                    val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, megolmBackupAuthData.signalableJSONDictionary())
+                    val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, signalableMegolmBackupAuthData.signalableJSONDictionary())
 
-                    val signedMegolmBackupAuthData = megolmBackupAuthData.copy(
+                    val signedMegolmBackupAuthData = MegolmBackupAuthData(
+                            publicKey = signalableMegolmBackupAuthData.publicKey,
+                            privateKeySalt = signalableMegolmBackupAuthData.privateKeySalt,
+                            privateKeyIterations = signalableMegolmBackupAuthData.privateKeyIterations,
                             signatures = objectSigner.signObject(canonicalJson)
                     )
 
@@ -223,8 +226,7 @@ internal class DefaultKeysBackupService @Inject constructor(
         @Suppress("UNCHECKED_CAST")
         val createKeysBackupVersionBody = CreateKeysBackupVersionBody(
                 algorithm = keysBackupCreationInfo.algorithm,
-                authData = MoshiProvider.providesMoshi().adapter(Map::class.java)
+                authData = keysBackupCreationInfo.authData.toJsonDict()
-                        .fromJson(keysBackupCreationInfo.authData?.toJsonString() ?: "") as JsonDict?
         )
 
         keysBackupStateManager.state = KeysBackupState.Enabling
@@ -234,7 +236,10 @@ internal class DefaultKeysBackupService @Inject constructor(
                     this.callback = object : MatrixCallback<KeysVersion> {
                         override fun onSuccess(data: KeysVersion) {
                             // Reset backup markers.
+                            cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+                                // move tx out of UI thread
                                 cryptoStore.resetBackupMarkers()
+                            }
 
                             val keyBackupVersion = KeysVersionResult(
                                     algorithm = createKeysBackupVersionBody.algorithm,
@@ -242,7 +247,7 @@ internal class DefaultKeysBackupService @Inject constructor(
                                     version = data.version,
                                     // We can consider that the server does not have keys yet
                                     count = 0,
-                                    hash = null
+                                    hash = ""
                             )
 
                             enableKeysBackup(keyBackupVersion)
@@ -264,7 +269,7 @@ internal class DefaultKeysBackupService @Inject constructor(
             withContext(coroutineDispatchers.crypto) {
                 // If we're currently backing up to this backup... stop.
                 // (We start using it automatically in createKeysBackupVersion so this is symmetrical).
-                if (keysBackupVersion != null && version == keysBackupVersion!!.version) {
+                if (keysBackupVersion != null && version == keysBackupVersion?.version) {
                     resetKeysBackupData()
                     keysBackupVersion = null
                     keysBackupStateManager.state = KeysBackupState.Unknown
@@ -405,10 +410,7 @@ internal class DefaultKeysBackupService @Inject constructor(
         val keysBackupVersionTrust = KeysBackupVersionTrust()
         val authData = keysBackupVersion.getAuthDataAsMegolmBackupAuthData()
 
-        if (keysBackupVersion.algorithm == null
+        if (authData == null || authData.publicKey.isEmpty() || authData.signatures.isEmpty()) {
-                || authData == null
-                || authData.publicKey.isEmpty()
-                || authData.signatures.isNullOrEmpty()) {
             Timber.v("getKeysBackupTrust: Key backup is absent or missing required data")
             return keysBackupVersionTrust
         }
@@ -476,7 +478,7 @@ internal class DefaultKeysBackupService @Inject constructor(
             cryptoCoroutineScope.launch(coroutineDispatchers.main) {
                 val updateKeysBackupVersionBody = withContext(coroutineDispatchers.crypto) {
                     // Get current signatures, or create an empty set
-                    val myUserSignatures = authData.signatures?.get(userId)?.toMutableMap() ?: HashMap()
+                    val myUserSignatures = authData.signatures[userId].orEmpty().toMutableMap()
 
                     if (trust) {
                         // Add current device signature
@@ -495,26 +497,23 @@ internal class DefaultKeysBackupService @Inject constructor(
                     // Create an updated version of KeysVersionResult
                     val newMegolmBackupAuthData = authData.copy()
 
-                    val newSignatures = newMegolmBackupAuthData.signatures!!.toMutableMap()
+                    val newSignatures = newMegolmBackupAuthData.signatures.toMutableMap()
                     newSignatures[userId] = myUserSignatures
 
                     val newMegolmBackupAuthDataWithNewSignature = newMegolmBackupAuthData.copy(
                             signatures = newSignatures
                     )
 
-                    val moshi = MoshiProvider.providesMoshi()
-                    val adapter = moshi.adapter(Map::class.java)
-
                     @Suppress("UNCHECKED_CAST")
                     UpdateKeysBackupVersionBody(
                             algorithm = keysBackupVersion.algorithm,
-                            authData = adapter.fromJson(newMegolmBackupAuthDataWithNewSignature.toJsonString()) as Map<String, Any>?,
+                            authData = newMegolmBackupAuthDataWithNewSignature.toJsonDict(),
-                            version = keysBackupVersion.version!!)
+                            version = keysBackupVersion.version)
                 }
 
                 // And send it to the homeserver
                 updateKeysBackupVersionTask
-                        .configureWith(UpdateKeysBackupVersionTask.Params(keysBackupVersion.version!!, updateKeysBackupVersionBody)) {
+                        .configureWith(UpdateKeysBackupVersionTask.Params(keysBackupVersion.version, updateKeysBackupVersionBody)) {
                             this.callback = object : MatrixCallback<Unit> {
                                 override fun onSuccess(data: Unit) {
                                     // Relaunch the state machine on this updated backup version
@@ -596,7 +595,9 @@ internal class DefaultKeysBackupService @Inject constructor(
                     val importResult = awaitCallback<ImportRoomKeysResult> {
                         restoreKeysWithRecoveryKey(keysBackupVersion, recoveryKey, null, null, null, it)
                     }
+                    withContext(coroutineDispatchers.crypto) {
                         cryptoStore.saveBackupRecoveryKey(recoveryKey, keysBackupVersion.version)
+                    }
                     Timber.i("onSecretKeyGossip: Recovered keys ${importResult.successfullyNumberOfImportedKeys} out of ${importResult.totalNumberOfKeys}")
                 } else {
                     Timber.e("onSecretKeyGossip: Recovery key is not valid ${keysBackupVersion.version}")
@@ -683,9 +684,9 @@ internal class DefaultKeysBackupService @Inject constructor(
                 stepProgressListener?.onStepProgress(StepProgressListener.Step.DownloadingKey)
 
                 // Get backed up keys from the homeserver
-                val data = getKeys(sessionId, roomId, keysVersionResult.version!!)
+                val data = getKeys(sessionId, roomId, keysVersionResult.version)
 
-                withContext(coroutineDispatchers.crypto) {
+                withContext(coroutineDispatchers.computation) {
                     val sessionsData = ArrayList<MegolmSessionData>()
                     // Restore that data
                     var sessionsFromHsCount = 0
@@ -1018,19 +1019,10 @@ internal class DefaultKeysBackupService @Inject constructor(
      * @return the authentication if found and valid, null in other case
      */
     private fun getMegolmBackupAuthData(keysBackupData: KeysVersionResult): MegolmBackupAuthData? {
-        if (keysBackupData.version.isNullOrBlank()
+        return keysBackupData
-                || keysBackupData.algorithm != MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
+                .takeIf { it.version.isNotEmpty() && it.algorithm == MXCRYPTO_ALGORITHM_MEGOLM_BACKUP }
-                || keysBackupData.authData == null) {
+                ?.getAuthDataAsMegolmBackupAuthData()
-            return null
+                ?.takeIf { it.publicKey.isNotEmpty() }
-        }
-
-        val authData = keysBackupData.getAuthDataAsMegolmBackupAuthData()
-
-        if (authData?.signatures == null || authData.publicKey.isBlank()) {
-            return null
-        }
-
-        return authData
     }
 
     /**
@@ -1118,12 +1110,13 @@ internal class DefaultKeysBackupService @Inject constructor(
      * @param keysVersionResult backup information object as returned by [getCurrentVersion].
      */
     private fun enableKeysBackup(keysVersionResult: KeysVersionResult) {
-        if (keysVersionResult.authData != null) {
         val retrievedMegolmBackupAuthData = keysVersionResult.getAuthDataAsMegolmBackupAuthData()
 
         if (retrievedMegolmBackupAuthData != null) {
             keysBackupVersion = keysVersionResult
+            cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
                 cryptoStore.setKeyBackupVersion(keysVersionResult.version)
+            }
 
             onServerDataRetrieved(keysVersionResult.count, keysVersionResult.hash)
 
@@ -1144,20 +1137,16 @@ internal class DefaultKeysBackupService @Inject constructor(
             Timber.e("Invalid authentication data")
             keysBackupStateManager.state = KeysBackupState.Disabled
         }
-        } else {
-            Timber.e("Invalid authentication data")
-            keysBackupStateManager.state = KeysBackupState.Disabled
-        }
     }
 
     /**
      * Update the DB with data fetch from the server
      */
-    private fun onServerDataRetrieved(count: Int?, hash: String?) {
+    private fun onServerDataRetrieved(count: Int?, etag: String?) {
         cryptoStore.setKeysBackupData(KeysBackupDataEntity()
                 .apply {
                     backupLastServerNumberOfKeys = count
-                    backupLastServerHash = hash
+                    backupLastServerHash = etag
                 }
         )
     }
@@ -1172,6 +1161,7 @@ internal class DefaultKeysBackupService @Inject constructor(
 
         cryptoStore.setKeyBackupVersion(null)
         cryptoStore.setKeysBackupData(null)
+        backupOlmPkEncryption?.releaseEncryption()
         backupOlmPkEncryption = null
 
         // Reset backup markers
@@ -1222,22 +1212,19 @@ internal class DefaultKeysBackupService @Inject constructor(
 
                 // Gather data to send to the homeserver
                 // roomId -> sessionId -> MXKeyBackupData
-                val keysBackupData = KeysBackupData(
+                val keysBackupData = KeysBackupData()
-                        roomIdToRoomKeysBackupData = HashMap()
-                )
 
-                for (olmInboundGroupSessionWrapper in olmInboundGroupSessionWrappers) {
+                olmInboundGroupSessionWrappers.forEach { olmInboundGroupSessionWrapper ->
-                    val keyBackupData = encryptGroupSession(olmInboundGroupSessionWrapper)
+                    val roomId = olmInboundGroupSessionWrapper.roomId ?: return@forEach
-                    if (keysBackupData.roomIdToRoomKeysBackupData[olmInboundGroupSessionWrapper.roomId] == null) {
+                    val olmInboundGroupSession = olmInboundGroupSessionWrapper.olmInboundGroupSession ?: return@forEach
-                        val roomKeysBackupData = RoomKeysBackupData(
-                                sessionIdToKeyBackupData = HashMap()
-                        )
-                        keysBackupData.roomIdToRoomKeysBackupData[olmInboundGroupSessionWrapper.roomId!!] = roomKeysBackupData
-                    }
 
                     try {
-                        keysBackupData.roomIdToRoomKeysBackupData[olmInboundGroupSessionWrapper.roomId]!!
+                        encryptGroupSession(olmInboundGroupSessionWrapper)
-                                .sessionIdToKeyBackupData[olmInboundGroupSessionWrapper.olmInboundGroupSession!!.sessionIdentifier()] = keyBackupData
+                                ?.let {
+                                    keysBackupData.roomIdToRoomKeysBackupData
+                                            .getOrPut(roomId) { RoomKeysBackupData() }
+                                            .sessionIdToKeyBackupData[olmInboundGroupSession.sessionIdentifier()] = it
+                                }
                     } catch (e: OlmException) {
                         Timber.e(e, "OlmException")
                     }
@@ -1245,7 +1232,12 @@ internal class DefaultKeysBackupService @Inject constructor(
 
                 Timber.v("backupKeys: 4 - Sending request")
 
-                val sendingRequestCallback = object : MatrixCallback<BackupKeysResult> {
+                // Make the request
+                val version = keysBackupVersion?.version ?: return@withContext
+
+                storeSessionDataTask
+                        .configureWith(StoreSessionsDataTask.Params(version, keysBackupData)) {
+                            this.callback = object : MatrixCallback<BackupKeysResult> {
                                 override fun onSuccess(data: BackupKeysResult) {
                                     uiHandler.post {
                                         Timber.v("backupKeys: 5a - Request complete")
@@ -1305,11 +1297,6 @@ internal class DefaultKeysBackupService @Inject constructor(
                                     }
                                 }
                             }
-
-                // Make the request
-                storeSessionDataTask
-                        .configureWith(StoreSessionsDataTask.Params(keysBackupVersion!!.version!!, keysBackupData)) {
-                            this.callback = sendingRequestCallback
                         }
                         .executeBy(taskExecutor)
             }
@@ -1318,47 +1305,45 @@ internal class DefaultKeysBackupService @Inject constructor(
 
     @VisibleForTesting
     @WorkerThread
-    fun encryptGroupSession(olmInboundGroupSessionWrapper: OlmInboundGroupSessionWrapper2): KeyBackupData {
+    fun encryptGroupSession(olmInboundGroupSessionWrapper: OlmInboundGroupSessionWrapper2): KeyBackupData? {
         // Gather information for each key
-        val device = cryptoStore.deviceWithIdentityKey(olmInboundGroupSessionWrapper.senderKey!!)
+        val device = olmInboundGroupSessionWrapper.senderKey?.let { cryptoStore.deviceWithIdentityKey(it) }
 
         // Build the m.megolm_backup.v1.curve25519-aes-sha2 data as defined at
         // https://github.com/uhoreg/matrix-doc/blob/e2e_backup/proposals/1219-storing-megolm-keys-serverside.md#mmegolm_backupv1curve25519-aes-sha2-key-format
-        val sessionData = olmInboundGroupSessionWrapper.exportKeys()
+        val sessionData = olmInboundGroupSessionWrapper.exportKeys() ?: return null
         val sessionBackupData = mapOf(
-                "algorithm" to sessionData!!.algorithm,
+                "algorithm" to sessionData.algorithm,
                 "sender_key" to sessionData.senderKey,
                 "sender_claimed_keys" to sessionData.senderClaimedKeys,
-                "forwarding_curve25519_key_chain" to (sessionData.forwardingCurve25519KeyChain
+                "forwarding_curve25519_key_chain" to (sessionData.forwardingCurve25519KeyChain.orEmpty()),
-                        ?: ArrayList<Any>()),
                 "session_key" to sessionData.sessionKey)
 
-        var encryptedSessionBackupData: OlmPkMessage? = null
+        val json = MoshiProvider.providesMoshi()
+                .adapter(Map::class.java)
+                .toJson(sessionBackupData)
 
-        val moshi = MoshiProvider.providesMoshi()
+        val encryptedSessionBackupData = try {
-        val adapter = moshi.adapter(Map::class.java)
+            backupOlmPkEncryption?.encrypt(json)
-
-        try {
-            val json = adapter.toJson(sessionBackupData)
-
-            encryptedSessionBackupData = backupOlmPkEncryption?.encrypt(json)
         } catch (e: OlmException) {
             Timber.e(e, "OlmException")
+            null
         }
+                ?: return null
 
         // Build backup data for that key
         return KeyBackupData(
                 firstMessageIndex = try {
-                    olmInboundGroupSessionWrapper.olmInboundGroupSession!!.firstKnownIndex
+                    olmInboundGroupSessionWrapper.olmInboundGroupSession?.firstKnownIndex ?: 0
                 } catch (e: OlmException) {
                     Timber.e(e, "OlmException")
                     0L
                 },
-                forwardedCount = olmInboundGroupSessionWrapper.forwardingCurve25519KeyChain!!.size,
+                forwardedCount = olmInboundGroupSessionWrapper.forwardingCurve25519KeyChain.orEmpty().size,
                 isVerified = device?.isVerified == true,
 
                 sessionData = mapOf(
-                        "ciphertext" to encryptedSessionBackupData!!.mCipherText,
+                        "ciphertext" to encryptedSessionBackupData.mCipherText,
                         "mac" to encryptedSessionBackupData.mMac,
                         "ephemeral" to encryptedSessionBackupData.mEphemeralKey)
         )
@@ -1371,9 +1356,9 @@ internal class DefaultKeysBackupService @Inject constructor(
 
         val jsonObject = keyBackupData.sessionData
 
-        val ciphertext = jsonObject?.get("ciphertext")?.toString()
+        val ciphertext = jsonObject["ciphertext"]?.toString()
-        val mac = jsonObject?.get("mac")?.toString()
+        val mac = jsonObject["mac"]?.toString()
-        val ephemeralKey = jsonObject?.get("ephemeral")?.toString()
+        val ephemeralKey = jsonObject["ephemeral"]?.toString()
 
         if (ciphertext != null && mac != null && ephemeralKey != null) {
             val encrypted = OlmPkMessage()
@@ -1418,8 +1403,7 @@ internal class DefaultKeysBackupService @Inject constructor(
         @Suppress("UNCHECKED_CAST")
         val createKeysBackupVersionBody = CreateKeysBackupVersionBody(
                 algorithm = keysBackupCreationInfo.algorithm,
-                authData = MoshiProvider.providesMoshi().adapter(Map::class.java)
+                authData = keysBackupCreationInfo.authData.toJsonDict()
-                        .fromJson(keysBackupCreationInfo.authData?.toJsonString() ?: "") as JsonDict?
         )
 
         createKeysBackupVersionTask

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/api/RoomKeysApi.kt

@@ -35,7 +35,7 @@ import retrofit2.http.Path
 import retrofit2.http.Query
 
 /**
- * Ref: https://github.com/uhoreg/matrix-doc/blob/e2e_backup/proposals/1219-storing-megolm-keys-serverside.md
+ * Ref: https://matrix.org/docs/spec/client_server/unstable#server-side-key-backups
  */
 internal interface RoomKeysApi {
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/MegolmBackupAuthData.kt

@@ -18,6 +18,7 @@ package org.matrix.android.sdk.internal.crypto.keysbackup.model
 
 import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
+import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.di.MoshiProvider
 
 /**
@@ -30,7 +31,7 @@ data class MegolmBackupAuthData(
          * The curve25519 public key used to encrypt the backups.
          */
         @Json(name = "public_key")
-        val publicKey: String = "",
+        val publicKey: String,
 
         /**
          * In case of a backup created from a password, the salt associated with the backup
@@ -50,20 +51,38 @@ data class MegolmBackupAuthData(
          * userId -> (deviceSignKeyId -> signature)
          */
         @Json(name = "signatures")
-        val signatures: Map<String, Map<String, String>>? = null
+        val signatures: Map<String, Map<String, String>>
 ) {
 
-    fun toJsonString(): String {
+    fun toJsonDict(): JsonDict {
-        return MoshiProvider.providesMoshi()
+        val moshi = MoshiProvider.providesMoshi()
+        val adapter = moshi.adapter(Map::class.java)
+
+        return moshi
                 .adapter(MegolmBackupAuthData::class.java)
                 .toJson(this)
+                .let {
+                    @Suppress("UNCHECKED_CAST")
+                    adapter.fromJson(it) as JsonDict
+                }
     }
 
-    /**
+    fun signalableJSONDictionary(): JsonDict {
-     * Same as the parent [MXJSONModel JSONDictionary] but return only
+        return SignalableMegolmBackupAuthData(
-     * data that must be signed.
+                publicKey = publicKey,
-     */
+                privateKeySalt = privateKeySalt,
-    fun signalableJSONDictionary(): Map<String, Any> = HashMap<String, Any>().apply {
+                privateKeyIterations = privateKeyIterations
+        )
+                .signalableJSONDictionary()
+    }
+}
+
+internal data class SignalableMegolmBackupAuthData(
+        val publicKey: String,
+        val privateKeySalt: String? = null,
+        val privateKeyIterations: Int? = null
+) {
+    fun signalableJSONDictionary(): JsonDict = HashMap<String, Any>().apply {
         put("public_key", publicKey)
 
         privateKeySalt?.let {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/MegolmBackupCreationInfo.kt

@@ -23,15 +23,15 @@ data class MegolmBackupCreationInfo(
         /**
          * The algorithm used for storing backups [org.matrix.androidsdk.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP].
          */
-        val algorithm: String = "",
+        val algorithm: String,
 
         /**
          * Authentication data.
          */
-        val authData: MegolmBackupAuthData? = null,
+        val authData: MegolmBackupAuthData,
 
         /**
          * The Base58 recovery key.
          */
-        val recoveryKey: String = ""
+        val recoveryKey: String
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/BackupKeysResult.kt

@@ -16,15 +16,16 @@
 
 package org.matrix.android.sdk.internal.crypto.keysbackup.model.rest
 
+import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
 
 @JsonClass(generateAdapter = true)
-data class BackupKeysResult(
+internal data class BackupKeysResult(
-
         // The hash value which is an opaque string representing stored keys in the backup
-        var hash: String? = null,
+        @Json(name = "etag")
+        val hash: String,
 
         // The number of keys stored in the backup.
-        var count: Int? = null
+        @Json(name = "count")
-
+        val count: Int
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/CreateKeysBackupVersionBody.kt

@@ -21,17 +21,17 @@ import com.squareup.moshi.JsonClass
 import org.matrix.android.sdk.api.util.JsonDict
 
 @JsonClass(generateAdapter = true)
-data class CreateKeysBackupVersionBody(
+internal data class CreateKeysBackupVersionBody(
         /**
          * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
          */
         @Json(name = "algorithm")
-        override val algorithm: String? = null,
+        override val algorithm: String,
 
         /**
          * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2"
          * see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
          */
         @Json(name = "auth_data")
-        override val authData: JsonDict? = null
+        override val authData: JsonDict
 ) : KeysAlgorithmAndData

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeyBackupData.kt

@@ -18,7 +18,7 @@ package org.matrix.android.sdk.internal.crypto.keysbackup.model.rest
 
 import com.squareup.moshi.Json
 import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.internal.di.MoshiProvider
+import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.internal.network.parsing.ForceToBoolean
 
 /**
@@ -30,13 +30,13 @@ data class KeyBackupData(
          * Required. The index of the first message in the session that the key can decrypt.
          */
         @Json(name = "first_message_index")
-        val firstMessageIndex: Long = 0,
+        val firstMessageIndex: Long,
 
         /**
          * Required. The number of times this key has been forwarded.
          */
         @Json(name = "forwarded_count")
-        val forwardedCount: Int = 0,
+        val forwardedCount: Int,
 
         /**
          * Whether the device backing up the key has verified the device that the key is from.
@@ -44,16 +44,11 @@ data class KeyBackupData(
          */
         @ForceToBoolean
         @Json(name = "is_verified")
-        val isVerified: Boolean = false,
+        val isVerified: Boolean,
 
         /**
          * Algorithm-dependent data.
          */
         @Json(name = "session_data")
-        val sessionData: Map<String, Any>? = null
+        val sessionData: JsonDict
-) {
+)
-
-    fun toJsonString(): String {
-        return MoshiProvider.providesMoshi().adapter(KeyBackupData::class.java).toJson(this)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeysAlgorithmAndData.kt

@@ -17,6 +17,7 @@
 package org.matrix.android.sdk.internal.crypto.keysbackup.model.rest
 
 import org.matrix.android.sdk.api.util.JsonDict
+import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.MegolmBackupAuthData
 import org.matrix.android.sdk.internal.di.MoshiProvider
 
@@ -37,24 +38,25 @@ import org.matrix.android.sdk.internal.di.MoshiProvider
  *     }
  * </pre>
  */
-interface KeysAlgorithmAndData {
+internal interface KeysAlgorithmAndData {
 
     /**
      * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
      */
-    val algorithm: String?
+    val algorithm: String
 
     /**
      * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2" see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
      */
-    val authData: JsonDict?
+    val authData: JsonDict
 
     /**
      * Facility method to convert authData to a MegolmBackupAuthData object
      */
     fun getAuthDataAsMegolmBackupAuthData(): MegolmBackupAuthData? {
         return MoshiProvider.providesMoshi()
-                .adapter(MegolmBackupAuthData::class.java)
+                .takeIf { algorithm == MXCRYPTO_ALGORITHM_MEGOLM_BACKUP }
-                .fromJsonValue(authData)
+                ?.adapter(MegolmBackupAuthData::class.java)
+                ?.fromJsonValue(authData)
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeysVersion.kt

@@ -23,5 +23,5 @@ import com.squareup.moshi.JsonClass
 data class KeysVersion(
         // the keys backup version
         @Json(name = "version")
-        val version: String? = null
+        val version: String
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/KeysVersionResult.kt

@@ -26,24 +26,24 @@ data class KeysVersionResult(
          * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
          */
         @Json(name = "algorithm")
-        override val algorithm: String? = null,
+        override val algorithm: String,
 
         /**
          * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2"
          * see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
          */
         @Json(name = "auth_data")
-        override val authData: JsonDict? = null,
+        override val authData: JsonDict,
 
         // the backup version
         @Json(name = "version")
-        val version: String? = null,
+        val version: String,
 
         // The hash value which is an opaque string representing stored keys in the backup
-        @Json(name = "hash")
+        @Json(name = "etag")
-        val hash: String? = null,
+        val hash: String,
 
         // The number of keys stored in the backup.
         @Json(name = "count")
-        val count: Int? = null
+        val count: Int
 ) : KeysAlgorithmAndData

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/model/rest/UpdateKeysBackupVersionBody.kt

@@ -26,16 +26,16 @@ data class UpdateKeysBackupVersionBody(
          * The algorithm used for storing backups. Currently, only "m.megolm_backup.v1.curve25519-aes-sha2" is defined
          */
         @Json(name = "algorithm")
-        override val algorithm: String? = null,
+        override val algorithm: String,
 
         /**
          * algorithm-dependent data, for "m.megolm_backup.v1.curve25519-aes-sha2"
          * see [org.matrix.android.sdk.internal.crypto.keysbackup.MegolmBackupAuthData]
          */
         @Json(name = "auth_data")
-        override val authData: JsonDict? = null,
+        override val authData: JsonDict,
 
-        // the backup version, mandatory
+        // Optional. The backup version. If present, must be the same as the path parameter.
         @Json(name = "version")
-        val version: String
+        val version: String? = null
 ) : KeysAlgorithmAndData

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/model/OlmInboundGroupSessionWrapper2.kt

@@ -48,17 +48,14 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
      */
     val firstKnownIndex: Long?
         get() {
-            if (null != olmInboundGroupSession) {
+            return try {
-                try {
+                olmInboundGroupSession?.firstKnownIndex
-                    return olmInboundGroupSession!!.firstKnownIndex
             } catch (e: Exception) {
                 Timber.e(e, "## getFirstKnownIndex() : getFirstKnownIndex failed")
+                null
             }
         }
 
-            return null
-        }
-
     /**
      * Constructor
      *
@@ -90,11 +87,13 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
     @Throws(Exception::class)
     constructor(megolmSessionData: MegolmSessionData) {
         try {
-            olmInboundGroupSession = OlmInboundGroupSession.importSession(megolmSessionData.sessionKey!!)
+            val safeSessionKey = megolmSessionData.sessionKey ?: throw Exception("invalid data")
-
+            olmInboundGroupSession = OlmInboundGroupSession.importSession(safeSessionKey)
-            if (olmInboundGroupSession!!.sessionIdentifier() != megolmSessionData.sessionId) {
+                    .also {
+                        if (it.sessionIdentifier() != megolmSessionData.sessionId) {
                             throw Exception("Mismatched group session Id")
                         }
+                    }
 
             senderKey = megolmSessionData.senderKey
             keysClaimed = megolmSessionData.senderClaimedKeys
@@ -120,16 +119,18 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
                 return null
             }
 
-            val wantedIndex = index ?: olmInboundGroupSession!!.firstKnownIndex
+            val safeOlmInboundGroupSession = olmInboundGroupSession ?: return null
+
+            val wantedIndex = index ?: safeOlmInboundGroupSession.firstKnownIndex
 
             MegolmSessionData(
                     senderClaimedEd25519Key = keysClaimed?.get("ed25519"),
-                    forwardingCurve25519KeyChain = ArrayList(forwardingCurve25519KeyChain!!),
+                    forwardingCurve25519KeyChain = forwardingCurve25519KeyChain?.toList().orEmpty(),
                     senderKey = senderKey,
                     senderClaimedKeys = keysClaimed,
                     roomId = roomId,
-                    sessionId = olmInboundGroupSession!!.sessionIdentifier(),
+                    sessionId = safeOlmInboundGroupSession.sessionIdentifier(),
-                    sessionKey = olmInboundGroupSession!!.export(wantedIndex),
+                    sessionKey = safeOlmInboundGroupSession.export(wantedIndex),
                     algorithm = MXCRYPTO_ALGORITHM_MEGOLM
             )
         } catch (e: Exception) {
@@ -145,14 +146,11 @@ class OlmInboundGroupSessionWrapper2 : Serializable {
      * @return the exported data
      */
     fun exportSession(messageIndex: Long): String? {
-        if (null != olmInboundGroupSession) {
+        return try {
-            try {
+            return olmInboundGroupSession?.export(messageIndex)
-                return olmInboundGroupSession!!.export(messageIndex)
         } catch (e: Exception) {
             Timber.e(e, "## exportSession() : export failed")
+            null
         }
     }
-
-        return null
-    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt

@@ -17,6 +17,7 @@
 package org.matrix.android.sdk.internal.crypto.store
 
 import androidx.lifecycle.LiveData
+import androidx.paging.PagedList
 import org.matrix.android.sdk.api.session.crypto.crosssigning.MXCrossSigningInfo
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.util.Optional
@@ -126,7 +127,10 @@ internal interface IMXCryptoStore {
     fun getPendingIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
 
     fun getPendingIncomingGossipingRequests(): List<IncomingShareRequestCommon>
+
     fun storeIncomingGossipingRequest(request: IncomingShareRequestCommon, ageLocalTS: Long?)
+
+    fun storeIncomingGossipingRequests(requests: List<IncomingShareRequestCommon>)
 //    fun getPendingIncomingSecretShareRequests(): List<IncomingSecretShareRequest>
 
     /**
@@ -364,6 +368,7 @@ internal interface IMXCryptoStore {
     fun getOrAddOutgoingSecretShareRequest(secretName: String, recipients: Map<String, List<String>>): OutgoingSecretRequest?
 
     fun saveGossipingEvent(event: Event)
+    fun saveGossipingEvents(events: List<Event>)
 
     fun updateGossipingRequestState(request: IncomingShareRequestCommon, state: GossipingRequestState) {
         updateGossipingRequestState(
@@ -441,11 +446,16 @@ internal interface IMXCryptoStore {
     // Dev tools
 
     fun getOutgoingRoomKeyRequests(): List<OutgoingRoomKeyRequest>
+    fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>>
     fun getOutgoingSecretKeyRequests(): List<OutgoingSecretRequest>
     fun getOutgoingSecretRequest(secretName: String): OutgoingSecretRequest?
     fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
-    fun getGossipingEventsTrail(): List<Event>
+    fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>>
+    fun getGossipingEventsTrail(): LiveData<PagedList<Event>>
+    fun getGossipingEvents(): List<Event>
 
     fun setDeviceKeysUploaded(uploaded: Boolean)
     fun getDeviceKeysUploaded(): Boolean
+    fun tidyUpDataBase()
+    fun logDbUsageInfo()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStore.kt

@@ -18,6 +18,8 @@ package org.matrix.android.sdk.internal.crypto.store.db
 
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.Transformations
+import androidx.paging.LivePagedListBuilder
+import androidx.paging.PagedList
 import com.zhuinden.monarchy.Monarchy
 import io.realm.Realm
 import io.realm.RealmConfiguration
@@ -62,6 +64,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoRoomEntityFie
 import org.matrix.android.sdk.internal.crypto.store.db.model.DeviceInfoEntity
 import org.matrix.android.sdk.internal.crypto.store.db.model.DeviceInfoEntityFields
 import org.matrix.android.sdk.internal.crypto.store.db.model.GossipingEventEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.GossipingEventEntityFields
 import org.matrix.android.sdk.internal.crypto.store.db.model.IncomingGossipingRequestEntity
 import org.matrix.android.sdk.internal.crypto.store.db.model.IncomingGossipingRequestEntityFields
 import org.matrix.android.sdk.internal.crypto.store.db.model.KeysBackupDataEntity
@@ -84,6 +87,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.query.get
 import org.matrix.android.sdk.internal.crypto.store.db.query.getById
 import org.matrix.android.sdk.internal.crypto.store.db.query.getOrCreate
 import org.matrix.android.sdk.internal.database.mapper.ContentMapper
+import org.matrix.android.sdk.internal.database.tools.RealmDebugTools
 import org.matrix.android.sdk.internal.di.CryptoDatabase
 import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.MoshiProvider
@@ -998,7 +1002,50 @@ internal class RealmCryptoStore @Inject constructor(
         }
     }
 
-    override fun getGossipingEventsTrail(): List<Event> {
+    override fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>> {
+        val realmDataSourceFactory = monarchy.createDataSourceFactory { realm ->
+            realm.where<IncomingGossipingRequestEntity>()
+                    .equalTo(IncomingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name)
+                    .sort(IncomingGossipingRequestEntityFields.LOCAL_CREATION_TIMESTAMP, Sort.DESCENDING)
+        }
+        val dataSourceFactory = realmDataSourceFactory.map {
+            it.toIncomingGossipingRequest() as? IncomingRoomKeyRequest
+                    ?: IncomingRoomKeyRequest(
+                            requestBody = null,
+                            deviceId = "",
+                            userId = "",
+                            requestId = "",
+                            state = GossipingRequestState.NONE,
+                            localCreationTimestamp = 0
+                    )
+        }
+        return monarchy.findAllPagedWithChanges(realmDataSourceFactory,
+                LivePagedListBuilder(dataSourceFactory,
+                        PagedList.Config.Builder()
+                                .setPageSize(20)
+                                .setEnablePlaceholders(false)
+                                .setPrefetchDistance(1)
+                                .build())
+        )
+    }
+
+    override fun getGossipingEventsTrail(): LiveData<PagedList<Event>> {
+        val realmDataSourceFactory = monarchy.createDataSourceFactory { realm ->
+            realm.where<GossipingEventEntity>().sort(GossipingEventEntityFields.AGE_LOCAL_TS, Sort.DESCENDING)
+        }
+        val dataSourceFactory = realmDataSourceFactory.map { it.toModel() }
+        val trail = monarchy.findAllPagedWithChanges(realmDataSourceFactory,
+                LivePagedListBuilder(dataSourceFactory,
+                        PagedList.Config.Builder()
+                                .setPageSize(20)
+                                .setEnablePlaceholders(false)
+                                .setPrefetchDistance(1)
+                                .build())
+        )
+        return trail
+    }
+
+    override fun getGossipingEvents(): List<Event> {
         return monarchy.fetchAllCopiedSync { realm ->
             realm.where<GossipingEventEntity>()
         }.map {
@@ -1066,7 +1113,28 @@ internal class RealmCryptoStore @Inject constructor(
         return request
     }
 
+    override fun saveGossipingEvents(events: List<Event>) {
+        val now = System.currentTimeMillis()
+        monarchy.writeAsync { realm ->
+            events.forEach { event ->
+                val ageLocalTs = event.unsignedData?.age?.let { now - it } ?: now
+                val entity = GossipingEventEntity(
+                        type = event.type,
+                        sender = event.senderId,
+                        ageLocalTs = ageLocalTs,
+                        content = ContentMapper.map(event.content)
+                ).apply {
+                    sendState = SendState.SYNCED
+                    decryptionResultJson = MoshiProvider.providesMoshi().adapter(OlmDecryptionResult::class.java).toJson(event.mxDecryptionResult)
+                    decryptionErrorCode = event.mCryptoError?.name
+                }
+                realm.insertOrUpdate(entity)
+            }
+        }
+    }
+
     override fun saveGossipingEvent(event: Event) {
+        monarchy.writeAsync { realm ->
             val now = System.currentTimeMillis()
             val ageLocalTs = event.unsignedData?.age?.let { now - it } ?: now
             val entity = GossipingEventEntity(
@@ -1076,14 +1144,12 @@ internal class RealmCryptoStore @Inject constructor(
                     content = ContentMapper.map(event.content)
             ).apply {
                 sendState = SendState.SYNCED
-            decryptionResultJson = MoshiProvider.providesMoshi().adapter<OlmDecryptionResult>(OlmDecryptionResult::class.java).toJson(event.mxDecryptionResult)
+                decryptionResultJson = MoshiProvider.providesMoshi().adapter(OlmDecryptionResult::class.java).toJson(event.mxDecryptionResult)
                 decryptionErrorCode = event.mCryptoError?.name
             }
-        doRealmTransaction(realmConfiguration) { realm ->
             realm.insertOrUpdate(entity)
         }
     }
-
 //    override fun getOutgoingRoomKeyRequestByState(states: Set<ShareRequestState>): OutgoingRoomKeyRequest? {
 //        val statesIndex = states.map { it.ordinal }.toTypedArray()
 //        return doRealmQueryAndCopy(realmConfiguration) { realm ->
@@ -1284,6 +1350,28 @@ internal class RealmCryptoStore @Inject constructor(
         }
     }
 
+    override fun storeIncomingGossipingRequests(requests: List<IncomingShareRequestCommon>) {
+        doRealmTransactionAsync(realmConfiguration) { realm ->
+            requests.forEach { request ->
+                // After a clear cache, we might have a
+                realm.createObject(IncomingGossipingRequestEntity::class.java).let {
+                    it.otherDeviceId = request.deviceId
+                    it.otherUserId = request.userId
+                    it.requestId = request.requestId ?: ""
+                    it.requestState = GossipingRequestState.PENDING
+                    it.localCreationTimestamp = request.localCreationTimestamp ?: System.currentTimeMillis()
+                    if (request is IncomingSecretShareRequest) {
+                        it.type = GossipRequestType.SECRET
+                        it.requestedInfoStr = request.secretName
+                    } else if (request is IncomingRoomKeyRequest) {
+                        it.type = GossipRequestType.KEY
+                        it.requestedInfoStr = request.requestBody?.toJson()
+                    }
+                }
+            }
+        }
+    }
+
 //    override fun getPendingIncomingSecretShareRequests(): List<IncomingSecretShareRequest> {
 //        return doRealmQueryAndCopyList(realmConfiguration) {
 //            it.where<GossipingEventEntity>()
@@ -1417,6 +1505,27 @@ internal class RealmCryptoStore @Inject constructor(
                 .filterNotNull()
     }
 
+    override fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>> {
+        val realmDataSourceFactory = monarchy.createDataSourceFactory { realm ->
+            realm
+                    .where(OutgoingGossipingRequestEntity::class.java)
+                    .equalTo(OutgoingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name)
+        }
+        val dataSourceFactory = realmDataSourceFactory.map {
+            it.toOutgoingGossipingRequest() as? OutgoingRoomKeyRequest
+                    ?: OutgoingRoomKeyRequest(requestBody = null, requestId = "?", recipients = emptyMap(), state = OutgoingGossipingRequestState.CANCELLED)
+        }
+        val trail = monarchy.findAllPagedWithChanges(realmDataSourceFactory,
+                LivePagedListBuilder(dataSourceFactory,
+                        PagedList.Config.Builder()
+                                .setPageSize(20)
+                                .setEnablePlaceholders(false)
+                                .setPrefetchDistance(1)
+                                .build())
+        )
+        return trail
+    }
+
     override fun getCrossSigningInfo(userId: String): MXCrossSigningInfo? {
         return doWithRealm(realmConfiguration) { realm ->
             val crossSigningInfo = realm.where(CrossSigningInfoEntity::class.java)
@@ -1558,4 +1667,48 @@ internal class RealmCryptoStore @Inject constructor(
             result
         }
     }
+
+    /**
+     * Some entries in the DB can get a bit out of control with time
+     * So we need to tidy up a bit
+     */
+    override fun tidyUpDataBase() {
+        val prevWeekTs = System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1_000
+        doRealmTransaction(realmConfiguration) { realm ->
+
+            // Only keep one week history
+            realm.where<IncomingGossipingRequestEntity>()
+                    .lessThan(IncomingGossipingRequestEntityFields.LOCAL_CREATION_TIMESTAMP, prevWeekTs)
+                    .findAll().let {
+                        Timber.i("## Crypto Clean up ${it.size} IncomingGossipingRequestEntity")
+                        it.deleteAllFromRealm()
+                    }
+
+            // Clean the cancelled ones?
+            realm.where<OutgoingGossipingRequestEntity>()
+                    .equalTo(OutgoingGossipingRequestEntityFields.REQUEST_STATE_STR, OutgoingGossipingRequestState.CANCELLED.name)
+                    .equalTo(OutgoingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name)
+                    .findAll().let {
+                        Timber.i("## Crypto Clean up ${it.size} OutgoingGossipingRequestEntity")
+                        it.deleteAllFromRealm()
+                    }
+
+            // Only keep one week history
+            realm.where<GossipingEventEntity>()
+                    .lessThan(GossipingEventEntityFields.AGE_LOCAL_TS, prevWeekTs)
+                    .findAll().let {
+                        Timber.i("## Crypto Clean up ${it.size} GossipingEventEntityFields")
+                        it.deleteAllFromRealm()
+                    }
+
+            // Can we do something for WithHeldSessionEntity?
+        }
+    }
+
+    /**
+     * Prints out database info
+     */
+    override fun logDbUsageInfo() {
+        RealmDebugTools(realmConfiguration).logInfo("Crypto")
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -43,6 +43,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.model.WithHeldSessionEnti
 import org.matrix.android.sdk.internal.di.SerializeNulls
 import io.realm.DynamicRealm
 import io.realm.RealmMigration
+import io.realm.RealmObjectSchema
 import org.matrix.androidsdk.crypto.data.MXOlmInboundGroupSession2
 import timber.log.Timber
 import javax.inject.Inject
@@ -57,6 +58,27 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
         const val CRYPTO_STORE_SCHEMA_VERSION = 11L
     }
 
+    private fun RealmObjectSchema.addFieldIfNotExists(fieldName: String, fieldType: Class<*>): RealmObjectSchema {
+        if (!hasField(fieldName)) {
+            addField(fieldName, fieldType)
+        }
+        return this
+    }
+
+    private fun RealmObjectSchema.removeFieldIfExists(fieldName: String): RealmObjectSchema {
+        if (hasField(fieldName)) {
+            removeField(fieldName)
+        }
+        return this
+    }
+
+    private fun RealmObjectSchema.setRequiredIfNotAlready(fieldName: String, isRequired: Boolean): RealmObjectSchema {
+        if (isRequired != isRequired(fieldName)) {
+            setRequired(fieldName, isRequired)
+        }
+        return this
+    }
+
     override fun migrate(realm: DynamicRealm, oldVersion: Long, newVersion: Long) {
         Timber.v("Migrating Realm Crypto from $oldVersion to $newVersion")
 
@@ -89,13 +111,13 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
         Timber.d("Update IncomingRoomKeyRequestEntity format: requestBodyString field is exploded into several fields")
 
         realm.schema.get("IncomingRoomKeyRequestEntity")
-                ?.addField("requestBodyAlgorithm", String::class.java)
+                ?.addFieldIfNotExists("requestBodyAlgorithm", String::class.java)
-                ?.addField("requestBodyRoomId", String::class.java)
+                ?.addFieldIfNotExists("requestBodyRoomId", String::class.java)
-                ?.addField("requestBodySenderKey", String::class.java)
+                ?.addFieldIfNotExists("requestBodySenderKey", String::class.java)
-                ?.addField("requestBodySessionId", String::class.java)
+                ?.addFieldIfNotExists("requestBodySessionId", String::class.java)
                 ?.transform { dynamicObject ->
-                    val requestBodyString = dynamicObject.getString("requestBodyString")
                     try {
+                        val requestBodyString = dynamicObject.getString("requestBodyString")
                         // It was a map before
                         val map: Map<String, String>? = deserializeFromRealm(requestBodyString)
 
@@ -109,18 +131,18 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                         Timber.e(e, "Error")
                     }
                 }
-                ?.removeField("requestBodyString")
+                ?.removeFieldIfExists("requestBodyString")
 
         Timber.d("Update IncomingRoomKeyRequestEntity format: requestBodyString field is exploded into several fields")
 
         realm.schema.get("OutgoingRoomKeyRequestEntity")
-                ?.addField("requestBodyAlgorithm", String::class.java)
+                ?.addFieldIfNotExists("requestBodyAlgorithm", String::class.java)
-                ?.addField("requestBodyRoomId", String::class.java)
+                ?.addFieldIfNotExists("requestBodyRoomId", String::class.java)
-                ?.addField("requestBodySenderKey", String::class.java)
+                ?.addFieldIfNotExists("requestBodySenderKey", String::class.java)
-                ?.addField("requestBodySessionId", String::class.java)
+                ?.addFieldIfNotExists("requestBodySessionId", String::class.java)
                 ?.transform { dynamicObject ->
-                    val requestBodyString = dynamicObject.getString("requestBodyString")
                     try {
+                        val requestBodyString = dynamicObject.getString("requestBodyString")
                         // It was a map before
                         val map: Map<String, String>? = deserializeFromRealm(requestBodyString)
 
@@ -134,10 +156,11 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                         Timber.e(e, "Error")
                     }
                 }
-                ?.removeField("requestBodyString")
+                ?.removeFieldIfExists("requestBodyString")
 
         Timber.d("Create KeysBackupDataEntity")
 
+        if (!realm.schema.contains("KeysBackupDataEntity")) {
             realm.schema.create("KeysBackupDataEntity")
                     .addField(KeysBackupDataEntityFields.PRIMARY_KEY, Integer::class.java)
                     .addPrimaryKey(KeysBackupDataEntityFields.PRIMARY_KEY)
@@ -145,14 +168,15 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
                     .addField(KeysBackupDataEntityFields.BACKUP_LAST_SERVER_HASH, String::class.java)
                     .addField(KeysBackupDataEntityFields.BACKUP_LAST_SERVER_NUMBER_OF_KEYS, Integer::class.java)
         }
+    }
 
     private fun migrateTo3RiotX(realm: DynamicRealm) {
         Timber.d("Step 2 -> 3")
         Timber.d("Migrate to RiotX model")
 
         realm.schema.get("CryptoRoomEntity")
-                ?.addField(CryptoRoomEntityFields.SHOULD_ENCRYPT_FOR_INVITED_MEMBERS, Boolean::class.java)
+                ?.addFieldIfNotExists(CryptoRoomEntityFields.SHOULD_ENCRYPT_FOR_INVITED_MEMBERS, Boolean::class.java)
-                ?.setRequired(CryptoRoomEntityFields.SHOULD_ENCRYPT_FOR_INVITED_MEMBERS, false)
+                ?.setRequiredIfNotAlready(CryptoRoomEntityFields.SHOULD_ENCRYPT_FOR_INVITED_MEMBERS, false)
 
         // Convert format of MXDeviceInfo, package has to be the same.
         realm.schema.get("DeviceInfoEntity")
@@ -204,8 +228,13 @@ internal class RealmCryptoStoreMigration @Inject constructor(private val crossSi
     // Version 4L added Cross Signing info persistence
     private fun migrateTo4(realm: DynamicRealm) {
         Timber.d("Step 3 -> 4")
-        Timber.d("Create KeyInfoEntity")
 
+        if (realm.schema.contains("TrustLevelEntity")) {
+            Timber.d("Skipping Step 3 -> 4 because entities already exist")
+            return
+        }
+
+        Timber.d("Create KeyInfoEntity")
         val trustLevelEntityEntitySchema = realm.schema.create("TrustLevelEntity")
                 .addField(TrustLevelEntityFields.CROSS_SIGNED_VERIFIED, Boolean::class.java)
                 .setNullable(TrustLevelEntityFields.CROSS_SIGNED_VERIFIED, true)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/tasks/EncryptEventTask.kt

@@ -17,8 +17,13 @@ package org.matrix.android.sdk.internal.crypto.tasks
 
 import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.toContent
 import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
+import org.matrix.android.sdk.internal.crypto.MXEventDecryptionResult
 import org.matrix.android.sdk.internal.crypto.model.MXEncryptEventContentResult
+import org.matrix.android.sdk.internal.database.mapper.ContentMapper
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
 import org.matrix.android.sdk.internal.task.Task
 import org.matrix.android.sdk.internal.util.awaitCallback
@@ -28,23 +33,23 @@ internal interface EncryptEventTask : Task<EncryptEventTask.Params, Event> {
     data class Params(val roomId: String,
                       val event: Event,
                       /**Do not encrypt these keys, keep them as is in encrypted content (e.g. m.relates_to)*/
-                      val keepKeys: List<String>? = null,
+                      val keepKeys: List<String>? = null
-                      val crypto: CryptoService
     )
 }
 
 internal class DefaultEncryptEventTask @Inject constructor(
-//        private val crypto: CryptoService
+        private val localEchoRepository: LocalEchoRepository,
-        private val localEchoRepository: LocalEchoRepository
+        private val cryptoService: CryptoService
 ) : EncryptEventTask {
     override suspend fun execute(params: EncryptEventTask.Params): Event {
-        if (!params.crypto.isRoomEncrypted(params.roomId)) return params.event
+        // don't want to wait for any query
+        // if (!params.crypto.isRoomEncrypted(params.roomId)) return params.event
         val localEvent = params.event
         if (localEvent.eventId == null) {
             throw IllegalArgumentException()
         }
 
-        localEchoRepository.updateSendState(localEvent.eventId, SendState.ENCRYPTING)
+        localEchoRepository.updateSendState(localEvent.eventId, localEvent.roomId, SendState.ENCRYPTING)
 
         val localMutableContent = localEvent.content?.toMutableMap() ?: mutableMapOf()
         params.keepKeys?.forEach {
@@ -52,8 +57,9 @@ internal class DefaultEncryptEventTask @Inject constructor(
         }
 
 //        try {
+        // let it throws
         awaitCallback<MXEncryptEventContentResult> {
-            params.crypto.encryptEventContent(localMutableContent, localEvent.type, params.roomId, it)
+            cryptoService.encryptEventContent(localMutableContent, localEvent.type, params.roomId, it)
         }.let { result ->
             val modifiedContent = HashMap(result.eventContent)
             params.keepKeys?.forEach { toKeep ->
@@ -63,18 +69,34 @@ internal class DefaultEncryptEventTask @Inject constructor(
                 }
             }
             val safeResult = result.copy(eventContent = modifiedContent)
+            // Better handling of local echo, to avoid decrypting transition on remote echo
+            // Should I only do it for text messages?
+            val decryptionLocalEcho = if (result.eventContent["algorithm"] == MXCRYPTO_ALGORITHM_MEGOLM) {
+                MXEventDecryptionResult(
+                        clearEvent = Event(
+                                type = localEvent.type,
+                                content = localEvent.content,
+                                roomId = localEvent.roomId
+                        ).toContent(),
+                        forwardingCurve25519KeyChain = emptyList(),
+                        senderCurve25519Key = result.eventContent["sender_key"] as? String,
+                        claimedEd25519Key = cryptoService.getMyDevice().fingerprint()
+                )
+            } else {
+                null
+            }
+
+            localEchoRepository.updateEcho(localEvent.eventId) { _, localEcho ->
+                localEcho.type = EventType.ENCRYPTED
+                localEcho.content = ContentMapper.map(modifiedContent)
+                decryptionLocalEcho?.also {
+                    localEcho.setDecryptionResult(it)
+                }
+            }
             return localEvent.copy(
                     type = safeResult.eventType,
                     content = safeResult.eventContent
             )
         }
-//        } catch (throwable: Throwable) {
-//            val sendState = when (throwable) {
-//                is Failure.CryptoError -> SendState.FAILED_UNKNOWN_DEVICES
-//                else                   -> SendState.UNDELIVERED
-//            }
-//            localEchoUpdater.updateSendState(localEvent.eventId, sendState)
-//            throw throwable
-//        }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/tasks/RedactEventTask.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.matrix.android.sdk.internal.crypto.tasks
+
+import org.greenrobot.eventbus.EventBus
+import org.matrix.android.sdk.internal.network.executeRequest
+import org.matrix.android.sdk.internal.session.room.RoomAPI
+import org.matrix.android.sdk.internal.session.room.send.SendResponse
+import org.matrix.android.sdk.internal.task.Task
+import javax.inject.Inject
+
+internal interface RedactEventTask : Task<RedactEventTask.Params, String> {
+    data class Params(
+            val txID: String,
+            val roomId: String,
+            val eventId: String,
+            val reason: String?
+    )
+}
+
+internal class DefaultRedactEventTask @Inject constructor(
+        private val roomAPI: RoomAPI,
+        private val eventBus: EventBus) : RedactEventTask {
+
+    override suspend fun execute(params: RedactEventTask.Params): String {
+        val executeRequest = executeRequest<SendResponse>(eventBus) {
+            apiCall = roomAPI.redactEvent(
+                    txId = params.txID,
+                    roomId = params.roomId,
+                    eventId = params.eventId,
+                    reason = if (params.reason == null) emptyMap() else mapOf("reason" to params.reason)
+            )
+        }
+        return executeRequest.eventId
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/tasks/SendEventTask.kt

@@ -15,7 +15,7 @@
  */
 package org.matrix.android.sdk.internal.crypto.tasks
 
-import org.matrix.android.sdk.api.session.crypto.CryptoService
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.internal.network.executeRequest
@@ -23,13 +23,12 @@ import org.matrix.android.sdk.internal.session.room.RoomAPI
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
 import org.matrix.android.sdk.internal.session.room.send.SendResponse
 import org.matrix.android.sdk.internal.task.Task
-import org.greenrobot.eventbus.EventBus
 import javax.inject.Inject
 
 internal interface SendEventTask : Task<SendEventTask.Params, String> {
     data class Params(
             val event: Event,
-            val cryptoService: CryptoService?
+            val encrypt: Boolean
     )
 }
 
@@ -40,11 +39,11 @@ internal class DefaultSendEventTask @Inject constructor(
         private val eventBus: EventBus) : SendEventTask {
 
     override suspend fun execute(params: SendEventTask.Params): String {
+        try {
             val event = handleEncryption(params)
             val localId = event.eventId!!
 
-        try {
+            localEchoRepository.updateSendState(localId, params.event.roomId, SendState.SENDING)
-            localEchoRepository.updateSendState(localId, SendState.SENDING)
             val executeRequest = executeRequest<SendResponse>(eventBus) {
                 apiCall = roomAPI.send(
                         localId,
@@ -53,26 +52,22 @@ internal class DefaultSendEventTask @Inject constructor(
                         eventType = event.type
                 )
             }
-            localEchoRepository.updateSendState(localId, SendState.SENT)
+            localEchoRepository.updateSendState(localId, params.event.roomId, SendState.SENT)
             return executeRequest.eventId
         } catch (e: Throwable) {
-            localEchoRepository.updateSendState(localId, SendState.UNDELIVERED)
+//            localEchoRepository.updateSendState(params.event.eventId!!, SendState.UNDELIVERED)
             throw e
         }
     }
 
+    @Throws
     private suspend fun handleEncryption(params: SendEventTask.Params): Event {
-        if (params.cryptoService?.isRoomEncrypted(params.event.roomId ?: "") == true) {
+        if (params.encrypt && !params.event.isEncrypted()) {
-            try {
             return encryptEventTask.execute(EncryptEventTask.Params(
                     params.event.roomId ?: "",
                     params.event,
-                        listOf("m.relates_to"),
+                    listOf("m.relates_to")
-                        params.cryptoService
             ))
-            } catch (throwable: Throwable) {
-                // We said it's ok to send verification request in clear
-            }
         }
         return params.event
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/tasks/SendVerificationMessageTask.kt

@@ -15,21 +15,20 @@
  */
 package org.matrix.android.sdk.internal.crypto.tasks
 
-import org.matrix.android.sdk.api.session.crypto.CryptoService
+import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.internal.crypto.CryptoSessionInfoProvider
 import org.matrix.android.sdk.internal.network.executeRequest
 import org.matrix.android.sdk.internal.session.room.RoomAPI
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
 import org.matrix.android.sdk.internal.session.room.send.SendResponse
 import org.matrix.android.sdk.internal.task.Task
-import org.greenrobot.eventbus.EventBus
 import javax.inject.Inject
 
 internal interface SendVerificationMessageTask : Task<SendVerificationMessageTask.Params, String> {
     data class Params(
-            val event: Event,
+            val event: Event
-            val cryptoService: CryptoService?
     )
 }
 
@@ -37,6 +36,7 @@ internal class DefaultSendVerificationMessageTask @Inject constructor(
         private val localEchoRepository: LocalEchoRepository,
         private val encryptEventTask: DefaultEncryptEventTask,
         private val roomAPI: RoomAPI,
+        private val cryptoSessionInfoProvider: CryptoSessionInfoProvider,
         private val eventBus: EventBus) : SendVerificationMessageTask {
 
     override suspend fun execute(params: SendVerificationMessageTask.Params): String {
@@ -44,7 +44,7 @@ internal class DefaultSendVerificationMessageTask @Inject constructor(
         val localId = event.eventId!!
 
         try {
-            localEchoRepository.updateSendState(localId, SendState.SENDING)
+            localEchoRepository.updateSendState(localId, event.roomId, SendState.SENDING)
             val executeRequest = executeRequest<SendResponse>(eventBus) {
                 apiCall = roomAPI.send(
                         localId,
@@ -53,22 +53,21 @@ internal class DefaultSendVerificationMessageTask @Inject constructor(
                         eventType = event.type
                 )
             }
-            localEchoRepository.updateSendState(localId, SendState.SENT)
+            localEchoRepository.updateSendState(localId, event.roomId, SendState.SENT)
             return executeRequest.eventId
         } catch (e: Throwable) {
-            localEchoRepository.updateSendState(localId, SendState.UNDELIVERED)
+            localEchoRepository.updateSendState(localId, event.roomId, SendState.UNDELIVERED)
             throw e
         }
     }
 
     private suspend fun handleEncryption(params: SendVerificationMessageTask.Params): Event {
-        if (params.cryptoService?.isRoomEncrypted(params.event.roomId ?: "") == true) {
+        if (cryptoSessionInfoProvider.isRoomEncrypted(params.event.roomId ?: "")) {
             try {
                 return encryptEventTask.execute(EncryptEventTask.Params(
                         params.event.roomId ?: "",
                         params.event,
-                        listOf("m.relates_to"),
+                        listOf("m.relates_to")
-                        params.cryptoService
                 ))
             } catch (throwable: Throwable) {
                 // We said it's ok to send verification request in clear

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/verification/DefaultVerificationService.kt

@@ -20,7 +20,6 @@ import android.os.Handler
 import android.os.Looper
 import dagger.Lazy
 import org.matrix.android.sdk.api.MatrixCallback
-import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.crypto.crosssigning.CrossSigningService
 import org.matrix.android.sdk.api.session.crypto.crosssigning.KEYBACKUP_SECRET_SSSS_NAME
 import org.matrix.android.sdk.api.session.crypto.crosssigning.MASTER_KEY_SSSS_NAME
@@ -111,9 +110,6 @@ internal class DefaultVerificationService @Inject constructor(
 
     private val uiHandler = Handler(Looper.getMainLooper())
 
-    // Cannot be injected in constructor as it creates a dependency cycle
-    lateinit var cryptoService: CryptoService
-
     // map [sender : [transaction]]
     private val txMap = HashMap<String, HashMap<String, DefaultVerificationTransaction>>()
 
@@ -129,7 +125,8 @@ internal class DefaultVerificationService @Inject constructor(
 
     // Event received from the sync
     fun onToDeviceEvent(event: Event) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+        Timber.d("## SAS onToDeviceEvent ${event.getClearType()}")
+        cryptoCoroutineScope.launch(coroutineDispatchers.dmVerif) {
             when (event.getClearType()) {
                 EventType.KEY_VERIFICATION_START         -> {
                     onStartRequestReceived(event)
@@ -163,7 +160,7 @@ internal class DefaultVerificationService @Inject constructor(
     }
 
     fun onRoomEvent(event: Event) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.dmVerif) {
             when (event.getClearType()) {
                 EventType.KEY_VERIFICATION_START  -> {
                     onRoomStartRequestReceived(event)
@@ -240,6 +237,7 @@ internal class DefaultVerificationService @Inject constructor(
     }
 
     private fun dispatchRequestAdded(tx: PendingVerificationRequest) {
+        Timber.v("## SAS dispatchRequestAdded txId:${tx.transactionId}")
         uiHandler.post {
             listeners.forEach {
                 try {
@@ -303,11 +301,14 @@ internal class DefaultVerificationService @Inject constructor(
         // We don't want to block here
         val otherDeviceId = validRequestInfo.fromDevice
 
+        Timber.v("## SAS onRequestReceived from $senderId and device $otherDeviceId, txId:${validRequestInfo.transactionId}")
+
         cryptoCoroutineScope.launch {
             if (checkKeysAreDownloaded(senderId, otherDeviceId) == null) {
                 Timber.e("## Verification device $otherDeviceId is not known")
             }
         }
+        Timber.v("## SAS onRequestReceived .. checkKeysAreDownloaded launched")
 
         // Remember this request
         val requestsForUser = pendingRequests.getOrPut(senderId) { mutableListOf() }
@@ -1203,7 +1204,9 @@ internal class DefaultVerificationService @Inject constructor(
         // TODO refactor this with the DM one
         Timber.i("## Requesting verification to user: $otherUserId with device list $otherDevices")
 
-        val targetDevices = otherDevices ?: cryptoService.getUserDevices(otherUserId).map { it.deviceId }
+        val targetDevices = otherDevices ?: cryptoStore.getUserDevices(otherUserId)
+                ?.values?.map { it.deviceId } ?: emptyList()
+
         val requestsForUser = pendingRequests.getOrPut(otherUserId) { mutableListOf() }
 
         val transport = verificationTransportToDeviceFactory.createTransport(null)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/verification/SendVerificationMessageWorker.kt

@@ -20,7 +20,6 @@ import androidx.work.Data
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
 import org.matrix.android.sdk.api.failure.shouldBeRetried
-import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.internal.crypto.tasks.SendVerificationMessageTask
 import org.matrix.android.sdk.internal.session.SessionComponent
 import org.matrix.android.sdk.internal.session.room.send.CancelSendTracker
@@ -47,7 +46,6 @@ internal class SendVerificationMessageWorker(context: Context,
 
     @Inject lateinit var sendVerificationMessageTask: SendVerificationMessageTask
     @Inject lateinit var localEchoRepository: LocalEchoRepository
-    @Inject lateinit var cryptoService: CryptoService
     @Inject lateinit var cancelSendTracker: CancelSendTracker
 
     override fun injectWith(injector: SessionComponent) {
@@ -70,8 +68,7 @@ internal class SendVerificationMessageWorker(context: Context,
         return try {
             val resultEventId = sendVerificationMessageTask.execute(
                     SendVerificationMessageTask.Params(
-                            event = localEvent,
+                            event = localEvent
-                            cryptoService = cryptoService
                     )
             )
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/verification/VerificationMessageProcessor.kt

@@ -15,7 +15,6 @@
  */
 package org.matrix.android.sdk.internal.crypto.verification
 
-import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
 import org.matrix.android.sdk.api.session.crypto.verification.VerificationService
 import org.matrix.android.sdk.api.session.events.model.Event
@@ -34,12 +33,13 @@ import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.session.EventInsertLiveProcessor
 import io.realm.Realm
+import org.matrix.android.sdk.internal.crypto.EventDecryptor
 import timber.log.Timber
 import java.util.ArrayList
 import javax.inject.Inject
 
 internal class VerificationMessageProcessor @Inject constructor(
-        private val cryptoService: CryptoService,
+        private val eventDecryptor: EventDecryptor,
         private val verificationService: DefaultVerificationService,
         @UserId private val userId: String,
         @DeviceId private val deviceId: String?
@@ -82,7 +82,7 @@ internal class VerificationMessageProcessor @Inject constructor(
             // TODO use a global event decryptor? attache to session and that listen to new sessionId?
             // for now decrypt sync
             try {
-                val result = cryptoService.decryptEvent(event, "")
+                val result = eventDecryptor.decryptEvent(event, "")
                 event.mxDecryptionResult = OlmDecryptionResult(
                         payload = result.clearEvent,
                         senderKey = result.senderCurve25519Key,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/EventInsertLiveObserver.kt

@@ -17,10 +17,6 @@
 package org.matrix.android.sdk.internal.database
 
 import com.zhuinden.monarchy.Monarchy
-import org.matrix.android.sdk.api.session.crypto.CryptoService
-import org.matrix.android.sdk.api.session.crypto.MXCryptoError
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.internal.crypto.algorithms.olm.OlmDecryptionResult
 import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.database.model.EventEntity
 import org.matrix.android.sdk.internal.database.model.EventInsertEntity
@@ -31,12 +27,13 @@ import org.matrix.android.sdk.internal.session.EventInsertLiveProcessor
 import io.realm.RealmConfiguration
 import io.realm.RealmResults
 import kotlinx.coroutines.launch
+import org.matrix.android.sdk.internal.crypto.EventDecryptor
 import timber.log.Timber
 import javax.inject.Inject
 
 internal class EventInsertLiveObserver @Inject constructor(@SessionDatabase realmConfiguration: RealmConfiguration,
                                                            private val processors: Set<@JvmSuppressWildcards EventInsertLiveProcessor>,
-                                                           private val cryptoService: CryptoService)
+                                                           private val eventDecryptor: EventDecryptor)
     : RealmLiveEntityObserver<EventInsertEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventInsertEntity> {
@@ -74,7 +71,7 @@ internal class EventInsertLiveObserver @Inject constructor(@SessionDatabase real
                         return@forEach
                     }
                     val domainEvent = event.asDomain()
-                    decryptIfNeeded(domainEvent)
+//                    decryptIfNeeded(domainEvent)
                     processors.filter {
                         it.shouldProcess(eventId, domainEvent.getClearType(), eventInsert.insertType)
                     }.forEach {
@@ -89,22 +86,22 @@ internal class EventInsertLiveObserver @Inject constructor(@SessionDatabase real
         }
     }
 
-    private fun decryptIfNeeded(event: Event) {
+//    private fun decryptIfNeeded(event: Event) {
-        if (event.isEncrypted() && event.mxDecryptionResult == null) {
+//        if (event.isEncrypted() && event.mxDecryptionResult == null) {
-            try {
+//            try {
-                val result = cryptoService.decryptEvent(event, event.roomId ?: "")
+//                val result = eventDecryptor.decryptEvent(event, event.roomId ?: "")
-                event.mxDecryptionResult = OlmDecryptionResult(
+//                event.mxDecryptionResult = OlmDecryptionResult(
-                        payload = result.clearEvent,
+//                        payload = result.clearEvent,
-                        senderKey = result.senderCurve25519Key,
+//                        senderKey = result.senderCurve25519Key,
-                        keysClaimed = result.claimedEd25519Key?.let { k -> mapOf("ed25519" to k) },
+//                        keysClaimed = result.claimedEd25519Key?.let { k -> mapOf("ed25519" to k) },
-                        forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
+//                        forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
-                )
+//                )
-            } catch (e: MXCryptoError) {
+//            } catch (e: MXCryptoError) {
-                Timber.v("Failed to decrypt event")
+//                Timber.v("Failed to decrypt event")
-                // TODO -> we should keep track of this and retry, or some processing will never be handled
+//                // TODO -> we should keep track of this and retry, or some processing will never be handled
-            }
+//            }
-        }
+//        }
-    }
+//    }
 
     private fun shouldProcess(eventInsertEntity: EventInsertEntity): Boolean {
         return processors.any {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/tools/RealmDebugTools.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database.tools
+
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import org.matrix.android.sdk.BuildConfig
+import timber.log.Timber
+
+internal class RealmDebugTools(
+        private val realmConfiguration: RealmConfiguration
+) {
+    /**
+     * Log info about the DB
+     */
+    fun logInfo(baseName: String) {
+        buildString {
+            append("\n$baseName Realm located at : ${realmConfiguration.realmDirectory}/${realmConfiguration.realmFileName}")
+
+            if (BuildConfig.LOG_PRIVATE_DATA) {
+                val key = realmConfiguration.encryptionKey.joinToString("") { byte -> "%02x".format(byte) }
+                append("\n$baseName Realm encryption key : $key")
+            }
+
+            Realm.getInstance(realmConfiguration).use { realm ->
+                // Check if we have data
+                separator()
+                separator()
+                append("\n$baseName Realm is empty: ${realm.isEmpty}")
+                var total = 0L
+                val maxNameLength = realmConfiguration.realmObjectClasses.maxOf { it.simpleName.length }
+                realmConfiguration.realmObjectClasses.forEach { modelClazz ->
+                    val count = realm.where(modelClazz).count()
+                    total += count
+                    append("\n$baseName Realm - count ${modelClazz.simpleName.padEnd(maxNameLength)} : $count")
+                }
+                separator()
+                append("\n$baseName Realm - total count: $total")
+                separator()
+                separator()
+            }
+        }
+                .let { Timber.i(it) }
+    }
+
+    private fun StringBuilder.separator() = append("\n==============================================")
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt

@@ -59,12 +59,13 @@ import org.matrix.android.sdk.api.session.user.UserService
 import org.matrix.android.sdk.api.session.widgets.WidgetService
 import org.matrix.android.sdk.internal.auth.SessionParamsStore
 import org.matrix.android.sdk.internal.crypto.DefaultCryptoService
+import org.matrix.android.sdk.internal.database.tools.RealmDebugTools
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.di.SessionId
 import org.matrix.android.sdk.internal.di.UnauthenticatedWithCertificate
 import org.matrix.android.sdk.internal.di.WorkManagerProvider
 import org.matrix.android.sdk.internal.session.identity.DefaultIdentityService
-import org.matrix.android.sdk.internal.session.room.timeline.TimelineEventDecryptor
+import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor
 import org.matrix.android.sdk.internal.session.sync.SyncTokenStore
 import org.matrix.android.sdk.internal.session.sync.job.SyncThread
 import org.matrix.android.sdk.internal.session.sync.job.SyncWorker
@@ -114,14 +115,14 @@ internal class DefaultSession @Inject constructor(
         private val accountDataService: Lazy<AccountDataService>,
         private val _sharedSecretStorageService: Lazy<SharedSecretStorageService>,
         private val accountService: Lazy<AccountService>,
-        private val timelineEventDecryptor: TimelineEventDecryptor,
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val defaultIdentityService: DefaultIdentityService,
         private val integrationManagerService: IntegrationManagerService,
         private val taskExecutor: TaskExecutor,
         private val callSignalingService: Lazy<CallSignalingService>,
         @UnauthenticatedWithCertificate
-        private val unauthenticatedWithCertificateOkHttpClient: Lazy<OkHttpClient>
+        private val unauthenticatedWithCertificateOkHttpClient: Lazy<OkHttpClient>,
+        private val eventSenderProcessor: EventSenderProcessor
 ) : Session,
         RoomService by roomService.get(),
         RoomDirectoryService by roomDirectoryService.get(),
@@ -160,7 +161,7 @@ internal class DefaultSession @Inject constructor(
             lifecycleObservers.forEach { it.onStart() }
         }
         eventBus.register(this)
-        timelineEventDecryptor.start()
+        eventSenderProcessor.start()
     }
 
     override fun requireBackgroundSync() {
@@ -197,13 +198,14 @@ internal class DefaultSession @Inject constructor(
     override fun close() {
         assert(isOpen)
         stopSync()
-        timelineEventDecryptor.destroy()
+        // timelineEventDecryptor.destroy()
         uiHandler.post {
             lifecycleObservers.forEach { it.onStop() }
         }
         cryptoService.get().close()
         isOpen = false
         eventBus.unregister(this)
+        eventSenderProcessor.interrupt()
     }
 
     override fun getSyncStateLive() = getSyncThread().liveState()
@@ -283,4 +285,8 @@ internal class DefaultSession @Inject constructor(
     override fun toString(): String {
         return "$myUserId - ${sessionParams.deviceId}"
     }
+
+    override fun logDbUsageInfo() {
+        RealmDebugTools(realmConfiguration).logInfo("Session")
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionComponent.kt

@@ -24,6 +24,7 @@ import org.matrix.android.sdk.internal.crypto.CancelGossipRequestWorker
 import org.matrix.android.sdk.internal.crypto.CryptoModule
 import org.matrix.android.sdk.internal.crypto.SendGossipRequestWorker
 import org.matrix.android.sdk.internal.crypto.SendGossipWorker
+import org.matrix.android.sdk.internal.crypto.crosssigning.UpdateTrustWorker
 import org.matrix.android.sdk.internal.crypto.verification.SendVerificationMessageWorker
 import org.matrix.android.sdk.internal.di.MatrixComponent
 import org.matrix.android.sdk.internal.di.SessionAssistedInjectModule
@@ -45,7 +46,6 @@ import org.matrix.android.sdk.internal.session.pushers.AddHttpPusherWorker
 import org.matrix.android.sdk.internal.session.pushers.PushersModule
 import org.matrix.android.sdk.internal.session.room.RoomModule
 import org.matrix.android.sdk.internal.session.room.relation.SendRelationWorker
-import org.matrix.android.sdk.internal.session.room.send.EncryptEventWorker
 import org.matrix.android.sdk.internal.session.room.send.MultipleEventSendingDispatcherWorker
 import org.matrix.android.sdk.internal.session.room.send.RedactEventWorker
 import org.matrix.android.sdk.internal.session.room.send.SendEventWorker
@@ -109,8 +109,6 @@ internal interface SessionComponent {
 
     fun inject(worker: SendRelationWorker)
 
-    fun inject(worker: EncryptEventWorker)
-
     fun inject(worker: MultipleEventSendingDispatcherWorker)
 
     fun inject(worker: RedactEventWorker)
@@ -131,6 +129,8 @@ internal interface SessionComponent {
 
     fun inject(worker: SendGossipWorker)
 
+    fun inject(worker: UpdateTrustWorker)
+
     @Component.Factory
     interface Factory {
         fun create(

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt

@@ -41,8 +41,9 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService
 import org.matrix.android.sdk.api.session.securestorage.SecureStorageService
 import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
 import org.matrix.android.sdk.api.session.typing.TypingUsersTracker
-import org.matrix.android.sdk.internal.crypto.crosssigning.ShieldTrustUpdater
 import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService
+import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask
+import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask
 import org.matrix.android.sdk.internal.crypto.verification.VerificationMessageProcessor
 import org.matrix.android.sdk.internal.database.DatabaseCleaner
 import org.matrix.android.sdk.internal.database.EventInsertLiveObserver
@@ -331,10 +332,6 @@ internal abstract class SessionModule {
     @IntoSet
     abstract fun bindWidgetUrlFormatter(formatter: DefaultWidgetURLFormatter): SessionLifecycleObserver
 
-    @Binds
-    @IntoSet
-    abstract fun bindShieldTrustUpdated(updater: ShieldTrustUpdater): SessionLifecycleObserver
-
     @Binds
     @IntoSet
     abstract fun bindIdentityService(service: DefaultIdentityService): SessionLifecycleObserver
@@ -367,4 +364,7 @@ internal abstract class SessionModule {
 
     @Binds
     abstract fun bindTypingUsersTracker(tracker: DefaultTypingUsersTracker): TypingUsersTracker
+
+    @Binds
+    abstract fun bindRedactEventTask(task: DefaultRedactEventTask): RedactEventTask
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt

@@ -36,8 +36,8 @@ import org.matrix.android.sdk.api.util.NoOpCancellable
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.session.SessionScope
 import org.matrix.android.sdk.internal.session.call.model.MxCallImpl
+import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory
-import org.matrix.android.sdk.internal.session.room.send.RoomEventSender
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.task.configureWith
 import timber.log.Timber
@@ -50,7 +50,7 @@ internal class DefaultCallSignalingService @Inject constructor(
         private val userId: String,
         private val activeCallHandler: ActiveCallHandler,
         private val localEchoEventFactory: LocalEchoEventFactory,
-        private val roomEventSender: RoomEventSender,
+        private val eventSenderProcessor: EventSenderProcessor,
         private val taskExecutor: TaskExecutor,
         private val turnServerTask: GetTurnServerTask
 ) : CallSignalingService {
@@ -103,7 +103,7 @@ internal class DefaultCallSignalingService @Inject constructor(
                 otherUserId = otherUserId,
                 isVideoCall = isVideoCall,
                 localEchoEventFactory = localEchoEventFactory,
-                roomEventSender = roomEventSender
+                eventSenderProcessor = eventSenderProcessor
         )
         activeCallHandler.addCall(call).also {
             return call
@@ -165,7 +165,7 @@ internal class DefaultCallSignalingService @Inject constructor(
                             otherUserId = event.senderId ?: return@let,
                             isVideoCall = content.isVideo(),
                             localEchoEventFactory = localEchoEventFactory,
-                            roomEventSender = roomEventSender
+                            eventSenderProcessor = eventSenderProcessor
                     )
                     activeCallHandler.addCall(incomingCall)
                     onCallInvite(incomingCall, content)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt

@@ -29,8 +29,8 @@ import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent
 import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent
 import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent
 import org.matrix.android.sdk.internal.session.call.DefaultCallSignalingService
+import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory
-import org.matrix.android.sdk.internal.session.room.send.RoomEventSender
 import org.webrtc.IceCandidate
 import org.webrtc.SessionDescription
 import timber.log.Timber
@@ -43,7 +43,7 @@ internal class MxCallImpl(
         override val otherUserId: String,
         override val isVideoCall: Boolean,
         private val localEchoEventFactory: LocalEchoEventFactory,
-        private val roomEventSender: RoomEventSender
+        private val eventSenderProcessor: EventSenderProcessor
 ) : MxCall {
 
     override var state: CallState = CallState.Idle
@@ -91,7 +91,7 @@ internal class MxCallImpl(
                 offer = CallInviteContent.Offer(sdp = sdp.description)
         )
                 .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) }
-                .also { roomEventSender.sendEvent(it) }
+                .also { eventSenderProcessor.postEvent(it) }
     }
 
     override fun sendLocalIceCandidates(candidates: List<IceCandidate>) {
@@ -106,7 +106,7 @@ internal class MxCallImpl(
                 }
         )
                 .let { createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = it.toContent()) }
-                .also { roomEventSender.sendEvent(it) }
+                .also { eventSenderProcessor.postEvent(it) }
     }
 
     override fun sendLocalIceCandidateRemovals(candidates: List<IceCandidate>) {
@@ -119,7 +119,7 @@ internal class MxCallImpl(
                 callId = callId
         )
                 .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) }
-                .also { roomEventSender.sendEvent(it) }
+                .also { eventSenderProcessor.postEvent(it) }
         state = CallState.Terminated
     }
 
@@ -132,7 +132,7 @@ internal class MxCallImpl(
                 answer = CallAnswerContent.Answer(sdp = sdp.description)
         )
                 .let { createEventAndLocalEcho(type = EventType.CALL_ANSWER, roomId = roomId, content = it.toContent()) }
-                .also { roomEventSender.sendEvent(it) }
+                .also { eventSenderProcessor.postEvent(it) }
     }
 
     private fun createEventAndLocalEcho(localId: String = LocalEcho.createLocalEchoId(), type: String, roomId: String, content: Content): Event {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/DefaultRoomService.kt

@@ -61,7 +61,7 @@ internal class DefaultRoomService @Inject constructor(
         return roomGetter.getRoom(roomId)
     }
 
-    override fun getExistingDirectRoomWithUser(otherUserId: String): Room? {
+    override fun getExistingDirectRoomWithUser(otherUserId: String): String? {
         return roomGetter.getDirectRoomWith(otherUserId)
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/EventRelationsAggregationProcessor.kt

@@ -15,7 +15,7 @@
  */
 package org.matrix.android.sdk.internal.session.room
 
-import org.matrix.android.sdk.api.session.crypto.CryptoService
+import io.realm.Realm
 import org.matrix.android.sdk.api.session.events.model.AggregatedAnnotation
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
@@ -47,7 +47,6 @@ import org.matrix.android.sdk.internal.database.query.getOrCreate
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.session.EventInsertLiveProcessor
-import io.realm.Realm
 import timber.log.Timber
 import javax.inject.Inject
 
@@ -78,9 +77,8 @@ private fun VerificationState?.toState(newState: VerificationState): Verificatio
     return newState
 }
 
-internal class EventRelationsAggregationProcessor @Inject constructor(@UserId private val userId: String,
+internal class EventRelationsAggregationProcessor @Inject constructor(@UserId private val userId: String)
-                                                                      private val cryptoService: CryptoService
+    : EventInsertLiveProcessor {
-) : EventInsertLiveProcessor {
 
     private val allowedTypes = listOf(
             EventType.MESSAGE,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/RoomGetter.kt

@@ -25,13 +25,12 @@ import org.matrix.android.sdk.internal.database.model.RoomSummaryEntity
 import org.matrix.android.sdk.internal.database.model.RoomSummaryEntityFields
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.session.SessionScope
-import org.matrix.android.sdk.internal.session.room.membership.RoomMemberHelper
 import javax.inject.Inject
 
 internal interface RoomGetter {
     fun getRoom(roomId: String): Room?
 
-    fun getDirectRoomWith(otherUserId: String): Room?
+    fun getDirectRoomWith(otherUserId: String): String?
 }
 
 @SessionScope
@@ -46,16 +45,14 @@ internal class DefaultRoomGetter @Inject constructor(
         }
     }
 
-    override fun getDirectRoomWith(otherUserId: String): Room? {
+    override fun getDirectRoomWith(otherUserId: String): String? {
         return realmSessionProvider.withRealm { realm ->
             RoomSummaryEntity.where(realm)
                     .equalTo(RoomSummaryEntityFields.IS_DIRECT, true)
                     .equalTo(RoomSummaryEntityFields.MEMBERSHIP_STR, Membership.JOIN.name)
                     .findAll()
-                    .filter { dm -> dm.otherMemberIds.contains(otherUserId) }
+                    .firstOrNull { dm -> dm.otherMemberIds.size == 1 && dm.otherMemberIds.first() == otherUserId }
-                    .map { it.roomId }
+                    ?.roomId
-                    .firstOrNull { roomId -> otherUserId in RoomMemberHelper(realm, roomId).getActiveRoomMemberIds() }
-                    ?.let { roomId -> createRoom(realm, roomId) }
         }
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/RoomModule.kt

@@ -101,6 +101,7 @@ internal abstract class RoomModule {
         fun providesHtmlRenderer(): HtmlRenderer {
             return HtmlRenderer
                     .builder()
+                    .softbreak("<br />")
                     .build()
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/create/CreateRoomBodyBuilder.kt

@@ -16,10 +16,10 @@
 
 package org.matrix.android.sdk.internal.session.room.create
 
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.crypto.crosssigning.CrossSigningService
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.toContent
 import org.matrix.android.sdk.api.session.identity.IdentityServiceError
 import org.matrix.android.sdk.api.session.identity.toMedium
 import org.matrix.android.sdk.api.session.room.model.create.CreateRoomParams
@@ -27,11 +27,13 @@ import org.matrix.android.sdk.internal.crypto.DeviceListManager
 import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import org.matrix.android.sdk.internal.di.AuthenticatedIdentity
 import org.matrix.android.sdk.internal.network.token.AccessTokenProvider
+import org.matrix.android.sdk.internal.session.content.FileUploader
 import org.matrix.android.sdk.internal.session.identity.EnsureIdentityTokenTask
 import org.matrix.android.sdk.internal.session.identity.data.IdentityStore
 import org.matrix.android.sdk.internal.session.identity.data.getIdentityServerUrlWithoutProtocol
 import org.matrix.android.sdk.internal.session.room.membership.threepid.ThreePidInviteBody
 import java.security.InvalidParameterException
+import java.util.UUID
 import javax.inject.Inject
 
 internal class CreateRoomBodyBuilder @Inject constructor(
@@ -39,6 +41,7 @@ internal class CreateRoomBodyBuilder @Inject constructor(
         private val crossSigningService: CrossSigningService,
         private val deviceListManager: DeviceListManager,
         private val identityStore: IdentityStore,
+        private val fileUploader: FileUploader,
         @AuthenticatedIdentity
         private val accessTokenProvider: AccessTokenProvider
 ) {
@@ -66,7 +69,8 @@ internal class CreateRoomBodyBuilder @Inject constructor(
 
         val initialStates = listOfNotNull(
                 buildEncryptionWithAlgorithmEvent(params),
-                buildHistoryVisibilityEvent(params)
+                buildHistoryVisibilityEvent(params),
+                buildAvatarEvent(params)
         )
                 .takeIf { it.isNotEmpty() }
 
@@ -85,15 +89,33 @@ internal class CreateRoomBodyBuilder @Inject constructor(
         )
     }
 
+    private suspend fun buildAvatarEvent(params: CreateRoomParams): Event? {
+        return params.avatarUri?.let { avatarUri ->
+            // First upload the image, ignoring any error
+            tryOrNull {
+                fileUploader.uploadFromUri(
+                        uri = avatarUri,
+                        filename = UUID.randomUUID().toString(),
+                        mimeType = "image/jpeg")
+            }
+                    ?.let { response ->
+                        Event(
+                                type = EventType.STATE_ROOM_AVATAR,
+                                stateKey = "",
+                                content = mapOf("url" to response.contentUri)
+                        )
+                    }
+        }
+    }
+
     private fun buildHistoryVisibilityEvent(params: CreateRoomParams): Event? {
         return params.historyVisibility
                 ?.let {
-                    val contentMap = mapOf("history_visibility" to it)
-
                     Event(
                             type = EventType.STATE_ROOM_HISTORY_VISIBILITY,
                             stateKey = "",
-                            content = contentMap.toContent())
+                            content = mapOf("history_visibility" to it)
+                    )
                 }
     }
 
@@ -111,12 +133,10 @@ internal class CreateRoomBodyBuilder @Inject constructor(
                     if (it != MXCRYPTO_ALGORITHM_MEGOLM) {
                         throw InvalidParameterException("Unsupported algorithm: $it")
                     }
-                    val contentMap = mapOf("algorithm" to it)
-
                     Event(
                             type = EventType.STATE_ROOM_ENCRYPTION,
                             stateKey = "",
-                            content = contentMap.toContent()
+                            content = mapOf("algorithm" to it)
                     )
                 }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/relation/DefaultRelationService.kt

@@ -17,12 +17,10 @@ package org.matrix.android.sdk.internal.session.room.relation
 
 import androidx.lifecycle.LiveData
 import androidx.lifecycle.Transformations
-import androidx.work.OneTimeWorkRequest
 import com.squareup.inject.assisted.Assisted
 import com.squareup.inject.assisted.AssistedInject
 import com.zhuinden.monarchy.Monarchy
 import org.matrix.android.sdk.api.MatrixCallback
-import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.room.model.EventAnnotationsSummary
 import org.matrix.android.sdk.api.session.room.model.message.MessageType
@@ -32,30 +30,25 @@ import org.matrix.android.sdk.api.util.Cancelable
 import org.matrix.android.sdk.api.util.NoOpCancellable
 import org.matrix.android.sdk.api.util.Optional
 import org.matrix.android.sdk.api.util.toOptional
+import org.matrix.android.sdk.internal.crypto.CryptoSessionInfoProvider
 import org.matrix.android.sdk.internal.database.mapper.TimelineEventMapper
 import org.matrix.android.sdk.internal.database.mapper.asDomain
 import org.matrix.android.sdk.internal.database.model.EventAnnotationsSummaryEntity
 import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
 import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.di.SessionDatabase
-import org.matrix.android.sdk.internal.di.SessionId
-import org.matrix.android.sdk.internal.session.room.send.EncryptEventWorker
 import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory
-import org.matrix.android.sdk.internal.session.room.send.RedactEventWorker
+import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor
-import org.matrix.android.sdk.internal.session.room.send.SendEventWorker
-import org.matrix.android.sdk.internal.session.room.timeline.TimelineSendEventWorkCommon
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.task.configureWith
 import org.matrix.android.sdk.internal.util.fetchCopyMap
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
 import timber.log.Timber
 
 internal class DefaultRelationService @AssistedInject constructor(
         @Assisted private val roomId: String,
-        @SessionId private val sessionId: String,
+        private val eventSenderProcessor: EventSenderProcessor,
-        private val timeLineSendEventWorkCommon: TimelineSendEventWorkCommon,
         private val eventFactory: LocalEchoEventFactory,
-        private val cryptoService: CryptoService,
+        private val cryptoSessionInfoProvider: CryptoSessionInfoProvider,
         private val findReactionEventForUndoTask: FindReactionEventForUndoTask,
         private val fetchEditHistoryTask: FetchEditHistoryTask,
         private val timelineEventMapper: TimelineEventMapper,
@@ -83,8 +76,7 @@ internal class DefaultRelationService @AssistedInject constructor(
                         .none { it.addedByMe && it.key == reaction }) {
             val event = eventFactory.createReactionEvent(roomId, targetEventId, reaction)
                     .also { saveLocalEcho(it) }
-            val sendRelationWork = createSendEventWork(event, true)
+            return eventSenderProcessor.postEvent(event, false /* reaction are not encrypted*/)
-            timeLineSendEventWorkCommon.postWork(roomId, sendRelationWork)
         } else {
             Timber.w("Reaction already added")
             NoOpCancellable
@@ -107,9 +99,7 @@ internal class DefaultRelationService @AssistedInject constructor(
                 data.redactEventId?.let { toRedact ->
                     val redactEvent = eventFactory.createRedactEvent(roomId, toRedact, null)
                             .also { saveLocalEcho(it) }
-                    val redactWork = createRedactEventWork(redactEvent, toRedact, null)
+                    eventSenderProcessor.postRedaction(redactEvent, null)
-
-                    timeLineSendEventWorkCommon.postWork(roomId, redactWork)
                 }
             }
         }
@@ -121,18 +111,6 @@ internal class DefaultRelationService @AssistedInject constructor(
                 .executeBy(taskExecutor)
     }
 
-    // TODO duplicate with send service?
-    private fun createRedactEventWork(localEvent: Event, eventId: String, reason: String?): OneTimeWorkRequest {
-        val sendContentWorkerParams = RedactEventWorker.Params(
-                sessionId,
-                localEvent.eventId!!,
-                roomId,
-                eventId,
-                reason)
-        val redactWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
-        return timeLineSendEventWorkCommon.createWork<RedactEventWorker>(redactWorkData, true)
-    }
-
     override fun editTextMessage(targetEventId: String,
                                  msgType: String,
                                  newBodyText: CharSequence,
@@ -141,14 +119,7 @@ internal class DefaultRelationService @AssistedInject constructor(
         val event = eventFactory
                 .createReplaceTextEvent(roomId, targetEventId, newBodyText, newBodyAutoMarkdown, msgType, compatibilityBodyText)
                 .also { saveLocalEcho(it) }
-        return if (cryptoService.isRoomEncrypted(roomId)) {
+        return eventSenderProcessor.postEvent(event, cryptoSessionInfoProvider.isRoomEncrypted(roomId))
-            val encryptWork = createEncryptEventWork(event, listOf("m.relates_to"))
-            val workRequest = createSendEventWork(event, false)
-            timeLineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, workRequest)
-        } else {
-            val workRequest = createSendEventWork(event, true)
-            timeLineSendEventWorkCommon.postWork(roomId, workRequest)
-        }
     }
 
     override fun editReply(replyToEdit: TimelineEvent,
@@ -165,18 +136,11 @@ internal class DefaultRelationService @AssistedInject constructor(
                 compatibilityBodyText
         )
                 .also { saveLocalEcho(it) }
-        return if (cryptoService.isRoomEncrypted(roomId)) {
+        return eventSenderProcessor.postEvent(event, cryptoSessionInfoProvider.isRoomEncrypted(roomId))
-            val encryptWork = createEncryptEventWork(event, listOf("m.relates_to"))
-            val workRequest = createSendEventWork(event, false)
-            timeLineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, workRequest)
-        } else {
-            val workRequest = createSendEventWork(event, true)
-            timeLineSendEventWorkCommon.postWork(roomId, workRequest)
-        }
     }
 
     override fun fetchEditHistory(eventId: String, callback: MatrixCallback<List<Event>>) {
-        val params = FetchEditHistoryTask.Params(roomId, cryptoService.isRoomEncrypted(roomId), eventId)
+        val params = FetchEditHistoryTask.Params(roomId, cryptoSessionInfoProvider.isRoomEncrypted(roomId), eventId)
         fetchEditHistoryTask
                 .configureWith(params) {
                     this.callback = callback
@@ -189,27 +153,7 @@ internal class DefaultRelationService @AssistedInject constructor(
                 ?.also { saveLocalEcho(it) }
                 ?: return null
 
-        return if (cryptoService.isRoomEncrypted(roomId)) {
+        return eventSenderProcessor.postEvent(event, cryptoSessionInfoProvider.isRoomEncrypted(roomId))
-            val encryptWork = createEncryptEventWork(event, listOf("m.relates_to"))
-            val workRequest = createSendEventWork(event, false)
-            timeLineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, workRequest)
-        } else {
-            val workRequest = createSendEventWork(event, true)
-            timeLineSendEventWorkCommon.postWork(roomId, workRequest)
-        }
-    }
-
-    private fun createEncryptEventWork(event: Event, keepKeys: List<String>?): OneTimeWorkRequest {
-        // Same parameter
-        val params = EncryptEventWorker.Params(sessionId, event.eventId!!, keepKeys)
-        val sendWorkData = WorkerParamsFactory.toData(params)
-        return timeLineSendEventWorkCommon.createWork<EncryptEventWorker>(sendWorkData, true)
-    }
-
-    private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, eventId = event.eventId!!)
-        val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
-        return timeLineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)
     }
 
     override fun getEventAnnotationsSummary(eventId: String): EventAnnotationsSummary? {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/DefaultSendService.kt

@@ -25,7 +25,6 @@ import com.squareup.inject.assisted.Assisted
 import com.squareup.inject.assisted.AssistedInject
 import kotlinx.coroutines.launch
 import org.matrix.android.sdk.api.session.content.ContentAttachmentData
-import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.isAttachmentMessage
 import org.matrix.android.sdk.api.session.events.model.isTextMessage
@@ -45,13 +44,13 @@ import org.matrix.android.sdk.api.util.Cancelable
 import org.matrix.android.sdk.api.util.CancelableBag
 import org.matrix.android.sdk.api.util.JsonDict
 import org.matrix.android.sdk.api.util.NoOpCancellable
+import org.matrix.android.sdk.internal.crypto.CryptoSessionInfoProvider
 import org.matrix.android.sdk.internal.di.SessionId
 import org.matrix.android.sdk.internal.di.WorkManagerProvider
 import org.matrix.android.sdk.internal.session.content.UploadContentWorker
-import org.matrix.android.sdk.internal.session.room.timeline.TimelineSendEventWorkCommon
+import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor
 import org.matrix.android.sdk.internal.task.TaskExecutor
 import org.matrix.android.sdk.internal.util.CancelableWork
-import org.matrix.android.sdk.internal.worker.AlwaysSuccessfulWorker
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
 import org.matrix.android.sdk.internal.worker.startChain
 import timber.log.Timber
@@ -63,13 +62,12 @@ private const val UPLOAD_WORK = "UPLOAD_WORK"
 internal class DefaultSendService @AssistedInject constructor(
         @Assisted private val roomId: String,
         private val workManagerProvider: WorkManagerProvider,
-        private val timelineSendEventWorkCommon: TimelineSendEventWorkCommon,
         @SessionId private val sessionId: String,
         private val localEchoEventFactory: LocalEchoEventFactory,
-        private val cryptoService: CryptoService,
+        private val cryptoSessionInfoProvider: CryptoSessionInfoProvider,
         private val taskExecutor: TaskExecutor,
         private val localEchoRepository: LocalEchoRepository,
-        private val roomEventSender: RoomEventSender,
+        private val eventSenderProcessor: EventSenderProcessor,
         private val cancelSendTracker: CancelSendTracker
 ) : SendService {
 
@@ -92,19 +90,6 @@ internal class DefaultSendService @AssistedInject constructor(
                 .let { sendEvent(it) }
     }
 
-    // For test only
-    private fun sendTextMessages(text: CharSequence, msgType: String, autoMarkdown: Boolean, times: Int): Cancelable {
-        return CancelableBag().apply {
-            // Send the event several times
-            repeat(times) { i ->
-                localEchoEventFactory.createTextEvent(roomId, msgType, "$text - $i", autoMarkdown)
-                        .also { createLocalEcho(it) }
-                        .let { sendEvent(it) }
-                        .also { add(it) }
-            }
-        }
-    }
-
     override fun sendFormattedTextMessage(text: String, formattedText: String, msgType: String): Cancelable {
         return localEchoEventFactory.createFormattedTextEvent(roomId, TextContent(text, formattedText), msgType)
                 .also { createLocalEcho(it) }
@@ -133,13 +118,14 @@ internal class DefaultSendService @AssistedInject constructor(
 
     override fun redactEvent(event: Event, reason: String?): Cancelable {
         // TODO manage media/attachements?
-        return createRedactEventWork(event, reason)
+        val redactionEcho = localEchoEventFactory.createRedactEvent(roomId, event.eventId!!, reason)
-                .let { timelineSendEventWorkCommon.postWork(roomId, it) }
+                .also { createLocalEcho(it) }
+        return eventSenderProcessor.postRedaction(redactionEcho, reason)
     }
 
     override fun resendTextMessage(localEcho: TimelineEvent): Cancelable {
         if (localEcho.root.isTextMessage() && localEcho.root.sendState.hasFailed()) {
-            localEchoRepository.updateSendState(localEcho.eventId, SendState.UNSENT)
+            localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
             return sendEvent(localEcho.root)
         }
         return NoOpCancellable
@@ -153,7 +139,7 @@ internal class DefaultSendService @AssistedInject constructor(
             val url = messageContent.getFileUrl() ?: return NoOpCancellable
             if (url.startsWith("mxc://")) {
                 // We need to resend only the message as the attachment is ok
-                localEchoRepository.updateSendState(localEcho.eventId, SendState.UNSENT)
+                localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
                 return sendEvent(localEcho.root)
             }
 
@@ -170,7 +156,7 @@ internal class DefaultSendService @AssistedInject constructor(
                             queryUri = Uri.parse(messageContent.url),
                             type = ContentAttachmentData.Type.IMAGE
                     )
-                    localEchoRepository.updateSendState(localEcho.eventId, SendState.UNSENT)
+                    localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
                     internalSendMedia(listOf(localEcho.root), attachmentData, true)
                 }
                 is MessageVideoContent -> {
@@ -184,7 +170,7 @@ internal class DefaultSendService @AssistedInject constructor(
                             queryUri = Uri.parse(messageContent.url),
                             type = ContentAttachmentData.Type.VIDEO
                     )
-                    localEchoRepository.updateSendState(localEcho.eventId, SendState.UNSENT)
+                    localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
                     internalSendMedia(listOf(localEcho.root), attachmentData, true)
                 }
                 is MessageFileContent  -> {
@@ -195,7 +181,7 @@ internal class DefaultSendService @AssistedInject constructor(
                             queryUri = Uri.parse(messageContent.url),
                             type = ContentAttachmentData.Type.FILE
                     )
-                    localEchoRepository.updateSendState(localEcho.eventId, SendState.UNSENT)
+                    localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
                     internalSendMedia(listOf(localEcho.root), attachmentData, true)
                 }
                 is MessageAudioContent -> {
@@ -207,7 +193,7 @@ internal class DefaultSendService @AssistedInject constructor(
                             queryUri = Uri.parse(messageContent.url),
                             type = ContentAttachmentData.Type.AUDIO
                     )
-                    localEchoRepository.updateSendState(localEcho.eventId, SendState.UNSENT)
+                    localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
                     internalSendMedia(listOf(localEcho.root), attachmentData, true)
                 }
                 else                   -> NoOpCancellable
@@ -222,25 +208,6 @@ internal class DefaultSendService @AssistedInject constructor(
         }
     }
 
-    override fun clearSendingQueue() {
-        timelineSendEventWorkCommon.cancelAllWorks(roomId)
-        workManagerProvider.workManager.cancelUniqueWork(buildWorkName(UPLOAD_WORK))
-
-        // Replace the worker chains with a AlwaysSuccessfulWorker, to ensure the queues are well emptied
-        workManagerProvider.matrixOneTimeWorkRequestBuilder<AlwaysSuccessfulWorker>()
-                .build().let {
-                    timelineSendEventWorkCommon.postWork(roomId, it, ExistingWorkPolicy.REPLACE)
-
-                    // need to clear also image sending queue
-                    workManagerProvider.workManager
-                            .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.REPLACE, it)
-                            .enqueue()
-                }
-        taskExecutor.executorScope.launch {
-            localEchoRepository.clearSendingQueue(roomId)
-        }
-    }
-
     override fun cancelSend(eventId: String) {
         cancelSendTracker.markLocalEchoForCancel(eventId, roomId)
         taskExecutor.executorScope.launch {
@@ -262,13 +229,6 @@ internal class DefaultSendService @AssistedInject constructor(
         }
     }
 
-//    override fun failAllPendingMessages() {
-//        taskExecutor.executorScope.launch {
-//            val eventsToResend = localEchoRepository.getAllEventsWithStates(roomId, SendState.PENDING_STATES)
-//            localEchoRepository.updateSendState(roomId, eventsToResend.map { it.eventId }, SendState.UNDELIVERED)
-//        }
-//    }
-
     override fun sendMedia(attachment: ContentAttachmentData,
                            compressBeforeSending: Boolean,
                            roomIds: Set<String>): Cancelable {
@@ -291,7 +251,7 @@ internal class DefaultSendService @AssistedInject constructor(
     private fun internalSendMedia(allLocalEchoes: List<Event>, attachment: ContentAttachmentData, compressBeforeSending: Boolean): Cancelable {
         val cancelableBag = CancelableBag()
 
-        allLocalEchoes.groupBy { cryptoService.isRoomEncrypted(it.roomId!!) }
+        allLocalEchoes.groupBy { cryptoSessionInfoProvider.isRoomEncrypted(it.roomId!!) }
                 .apply {
                     keys.forEach { isRoomEncrypted ->
                         // Should never be empty
@@ -301,7 +261,7 @@ internal class DefaultSendService @AssistedInject constructor(
                         val dispatcherWork = createMultipleEventDispatcherWork(isRoomEncrypted)
 
                         workManagerProvider.workManager
-                                .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.APPEND, uploadWork)
+                                .beginUniqueWork(buildWorkName(UPLOAD_WORK), ExistingWorkPolicy.APPEND_OR_REPLACE, uploadWork)
                                 .then(dispatcherWork)
                                 .enqueue()
                                 .also { operation ->
@@ -322,7 +282,7 @@ internal class DefaultSendService @AssistedInject constructor(
     }
 
     private fun sendEvent(event: Event): Cancelable {
-        return roomEventSender.sendEvent(event)
+        return eventSenderProcessor.postEvent(event, cryptoSessionInfoProvider.isRoomEncrypted(event.roomId!!))
     }
 
     private fun createLocalEcho(event: Event) {
@@ -333,28 +293,6 @@ internal class DefaultSendService @AssistedInject constructor(
         return "${roomId}_$identifier"
     }
 
-    private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        // Same parameter
-        return EncryptEventWorker.Params(sessionId, event.eventId ?: "")
-                .let { WorkerParamsFactory.toData(it) }
-                .let {
-                    workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
-                            .setConstraints(WorkManagerProvider.workConstraints)
-                            .setInputData(it)
-                            .startChain(startChain)
-                            .setBackoffCriteria(BackoffPolicy.LINEAR, WorkManagerProvider.BACKOFF_DELAY, TimeUnit.MILLISECONDS)
-                            .build()
-                }
-    }
-
-    private fun createRedactEventWork(event: Event, reason: String?): OneTimeWorkRequest {
-        return localEchoEventFactory.createRedactEvent(roomId, event.eventId!!, reason)
-                .also { createLocalEcho(it) }
-                .let { RedactEventWorker.Params(sessionId, it.eventId!!, roomId, event.eventId, reason) }
-                .let { WorkerParamsFactory.toData(it) }
-                .let { timelineSendEventWorkCommon.createWork<RedactEventWorker>(it, true) }
-    }
-
     private fun createUploadMediaWork(allLocalEchos: List<Event>,
                                       attachment: ContentAttachmentData,
                                       isRoomEncrypted: Boolean,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/EncryptEventWorker.kt

@@ -1,146 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.session.room.send
-
-import android.content.Context
-import androidx.work.WorkerParameters
-import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.api.failure.Failure
-import org.matrix.android.sdk.api.session.crypto.CryptoService
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.toContent
-import org.matrix.android.sdk.api.session.room.send.SendState
-import org.matrix.android.sdk.internal.crypto.MXCRYPTO_ALGORITHM_MEGOLM
-import org.matrix.android.sdk.internal.crypto.MXEventDecryptionResult
-import org.matrix.android.sdk.internal.crypto.model.MXEncryptEventContentResult
-import org.matrix.android.sdk.internal.database.mapper.ContentMapper
-import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.util.awaitCallback
-import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import timber.log.Timber
-import javax.inject.Inject
-
-/**
- * Possible previous worker: None
- * Possible next worker    : Always [SendEventWorker]
- */
-internal class EncryptEventWorker(context: Context, params: WorkerParameters)
-    : SessionSafeCoroutineWorker<EncryptEventWorker.Params>(context, params, Params::class.java) {
-
-    @JsonClass(generateAdapter = true)
-    internal data class Params(
-            override val sessionId: String,
-            val eventId: String,
-            /** Do not encrypt these keys, keep them as is in encrypted content (e.g. m.relates_to) */
-            val keepKeys: List<String>? = null,
-            override val lastFailureMessage: String? = null
-    ) : SessionWorkerParams
-
-    @Inject lateinit var crypto: CryptoService
-    @Inject lateinit var localEchoRepository: LocalEchoRepository
-    @Inject lateinit var cancelSendTracker: CancelSendTracker
-
-    override fun injectWith(injector: SessionComponent) {
-        injector.inject(this)
-    }
-
-    override suspend fun doSafeWork(params: Params): Result {
-        Timber.v("## SendEvent: Start Encrypt work for event ${params.eventId}")
-
-        val localEvent = localEchoRepository.getUpToDateEcho(params.eventId)
-        if (localEvent?.eventId == null) {
-            return Result.success()
-        }
-
-        if (cancelSendTracker.isCancelRequestedFor(localEvent.eventId, localEvent.roomId)) {
-            return Result.success()
-                    .also { Timber.e("## SendEvent: Event sending has been cancelled ${localEvent.eventId}") }
-        }
-
-        localEchoRepository.updateSendState(localEvent.eventId, SendState.ENCRYPTING)
-
-        val localMutableContent = localEvent.content?.toMutableMap() ?: mutableMapOf()
-        params.keepKeys?.forEach {
-            localMutableContent.remove(it)
-        }
-
-        var error: Throwable? = null
-        var result: MXEncryptEventContentResult? = null
-        try {
-            result = awaitCallback {
-                crypto.encryptEventContent(localMutableContent, localEvent.type, localEvent.roomId!!, it)
-            }
-        } catch (throwable: Throwable) {
-            error = throwable
-        }
-        if (result != null) {
-            val modifiedContent = HashMap(result.eventContent)
-            params.keepKeys?.forEach { toKeep ->
-                localEvent.content?.get(toKeep)?.let {
-                    // put it back in the encrypted thing
-                    modifiedContent[toKeep] = it
-                }
-            }
-            // Better handling of local echo, to avoid decrypting transition on remote echo
-            // Should I only do it for text messages?
-            val decryptionLocalEcho = if (result.eventContent["algorithm"] == MXCRYPTO_ALGORITHM_MEGOLM) {
-                MXEventDecryptionResult(
-                        clearEvent = Event(
-                                type = localEvent.type,
-                                content = localEvent.content,
-                                roomId = localEvent.roomId
-                        ).toContent(),
-                        forwardingCurve25519KeyChain = emptyList(),
-                        senderCurve25519Key = result.eventContent["sender_key"] as? String,
-                        claimedEd25519Key = crypto.getMyDevice().fingerprint()
-                )
-            } else {
-                null
-            }
-            localEchoRepository.updateEcho(localEvent.eventId) { _, localEcho ->
-                localEcho.type = EventType.ENCRYPTED
-                localEcho.content = ContentMapper.map(modifiedContent)
-                decryptionLocalEcho?.also {
-                    localEcho.setDecryptionResult(it)
-                }
-            }
-
-            val nextWorkerParams = SendEventWorker.Params(sessionId = params.sessionId, eventId = params.eventId)
-            return Result.success(WorkerParamsFactory.toData(nextWorkerParams))
-        } else {
-            val sendState = when (error) {
-                is Failure.CryptoError -> SendState.FAILED_UNKNOWN_DEVICES
-                else                   -> SendState.UNDELIVERED
-            }
-            localEchoRepository.updateSendState(localEvent.eventId, sendState)
-            // always return success, or the chain will be stuck for ever!
-            val nextWorkerParams = SendEventWorker.Params(
-                    sessionId = params.sessionId,
-                    eventId = localEvent.eventId,
-                    lastFailureMessage = error?.localizedMessage ?: "Error"
-            )
-            return Result.success(WorkerParamsFactory.toData(nextWorkerParams))
-        }
-    }
-
-    override fun buildErrorParams(params: Params, message: String): Params {
-        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoEventFactory.kt

@@ -90,8 +90,7 @@ internal class LocalEchoEventFactory @Inject constructor(
 
     private fun createTextContent(text: CharSequence, autoMarkdown: Boolean): TextContent {
         if (autoMarkdown) {
-            val source = textPillsUtils.processSpecialSpansToMarkdown(text) ?: text.toString()
+            return markdownParser.parse(text)
-            return markdownParser.parse(source)
         } else {
             // Try to detect pills
             textPillsUtils.processSpecialSpansToHtml(text)?.let {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/LocalEchoRepository.kt

@@ -88,8 +88,9 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
         }
     }
 
-    fun updateSendState(eventId: String, sendState: SendState) {
+    fun updateSendState(eventId: String, roomId: String?, sendState: SendState) {
         Timber.v("## SendEvent: [${System.currentTimeMillis()}] Update local state of $eventId to ${sendState.name}")
+        eventBus.post(DefaultTimeline.OnLocalEchoUpdated(roomId ?: "", eventId, sendState))
         updateEchoAsync(eventId) { realm, sendingEventEntity ->
             if (sendState == SendState.SENT && sendingEventEntity.sendState == SendState.SYNCED) {
                 // If already synced, do not put as sent
@@ -137,6 +138,14 @@ internal class LocalEchoRepository @Inject constructor(@SessionDatabase private
         }
     }
 
+     fun deleteFailedEchoAsync(roomId: String, eventId: String?) {
+        monarchy.runTransactionSync { realm ->
+            TimelineEventEntity.where(realm, roomId = roomId, eventId = eventId ?: "").findFirst()?.deleteFromRealm()
+            EventEntity.where(realm, eventId = eventId ?: "").findFirst()?.deleteFromRealm()
+            roomSummaryUpdater.updateSendingInformation(realm, roomId)
+        }
+    }
+
     suspend fun clearSendingQueue(roomId: String) {
         monarchy.awaitTransaction { realm ->
             TimelineEventEntity

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/MarkdownParser.kt

@@ -18,6 +18,7 @@ package org.matrix.android.sdk.internal.session.room.send
 
 import org.commonmark.parser.Parser
 import org.commonmark.renderer.html.HtmlRenderer
+import org.matrix.android.sdk.internal.session.room.send.pills.TextPillsUtils
 import javax.inject.Inject
 
 /**
@@ -27,18 +28,21 @@ import javax.inject.Inject
  */
 internal class MarkdownParser @Inject constructor(
         private val parser: Parser,
-        private val htmlRenderer: HtmlRenderer
+        private val htmlRenderer: HtmlRenderer,
+        private val textPillsUtils: TextPillsUtils
 ) {
 
     private val mdSpecialChars = "[`_\\-*>.\\[\\]#~]".toRegex()
 
-    fun parse(text: String): TextContent {
+    fun parse(text: CharSequence): TextContent {
+        val source = textPillsUtils.processSpecialSpansToMarkdown(text) ?: text.toString()
+
         // If no special char are detected, just return plain text
-        if (text.contains(mdSpecialChars).not()) {
+        if (source.contains(mdSpecialChars).not()) {
-            return TextContent(text)
+            return TextContent(source)
         }
 
-        val document = parser.parse(text)
+        val document = parser.parse(source)
         val htmlText = htmlRenderer.render(document)
 
         // Cleanup extra paragraph
@@ -48,13 +52,14 @@ internal class MarkdownParser @Inject constructor(
             htmlText
         }
 
-        return if (isFormattedTextPertinent(text, cleanHtmlText)) {
+        return if (isFormattedTextPertinent(source, cleanHtmlText)) {
             // According to https://matrix.org/docs/spec/client_server/latest#m-room-message-msgtypes:
             // The plain text version of the HTML should be provided in the body.
             // But it caused too many problems so it has been removed in #2002
-            TextContent(text, cleanHtmlText.postTreatment())
+            // See #739
+            TextContent(text.toString(), cleanHtmlText.postTreatment())
         } else {
-            TextContent(text)
+            TextContent(source)
         }
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/MultipleEventSendingDispatcherWorker.kt

@@ -17,7 +17,6 @@
 package org.matrix.android.sdk.internal.session.room.send
 
 import android.content.Context
-import androidx.work.BackoffPolicy
 import androidx.work.OneTimeWorkRequest
 import androidx.work.WorkerParameters
 import com.squareup.moshi.JsonClass
@@ -31,7 +30,6 @@ import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
 import org.matrix.android.sdk.internal.worker.startChain
 import timber.log.Timber
-import java.util.concurrent.TimeUnit
 import javax.inject.Inject
 
 /**
@@ -57,7 +55,7 @@ internal class MultipleEventSendingDispatcherWorker(context: Context, params: Wo
 
     override fun doOnError(params: Params): Result {
         params.localEchoIds.forEach { localEchoIds ->
-            localEchoRepository.updateSendState(localEchoIds.eventId, SendState.UNDELIVERED)
+            localEchoRepository.updateSendState(localEchoIds.eventId, localEchoIds.roomId, SendState.UNDELIVERED)
         }
 
         return super.doOnError(params)
@@ -73,20 +71,11 @@ internal class MultipleEventSendingDispatcherWorker(context: Context, params: Wo
         params.localEchoIds.forEach { localEchoIds ->
             val roomId = localEchoIds.roomId
             val eventId = localEchoIds.eventId
-            if (params.isEncrypted) {
+                localEchoRepository.updateSendState(eventId, roomId, SendState.SENDING)
-                localEchoRepository.updateSendState(eventId, SendState.ENCRYPTING)
-                Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule encrypt and send event $eventId")
-                val encryptWork = createEncryptEventWork(params.sessionId, eventId, true)
-                // Note that event will be replaced by the result of the previous work
-                val sendWork = createSendEventWork(params.sessionId, eventId, false)
-                timelineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, sendWork)
-            } else {
-                localEchoRepository.updateSendState(eventId, SendState.SENDING)
                 Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule send event $eventId")
                 val sendWork = createSendEventWork(params.sessionId, eventId, true)
                 timelineSendEventWorkCommon.postWork(roomId, sendWork)
         }
-        }
 
         return Result.success()
     }
@@ -95,18 +84,6 @@ internal class MultipleEventSendingDispatcherWorker(context: Context, params: Wo
         return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
     }
 
-    private fun createEncryptEventWork(sessionId: String, eventId: String, startChain: Boolean): OneTimeWorkRequest {
-        val params = EncryptEventWorker.Params(sessionId, eventId)
-        val sendWorkData = WorkerParamsFactory.toData(params)
-
-        return workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
-                .setConstraints(WorkManagerProvider.workConstraints)
-                .setInputData(sendWorkData)
-                .startChain(startChain)
-                .setBackoffCriteria(BackoffPolicy.LINEAR, WorkManagerProvider.BACKOFF_DELAY, TimeUnit.MILLISECONDS)
-                .build()
-    }
-
     private fun createSendEventWork(sessionId: String, eventId: String, startChain: Boolean): OneTimeWorkRequest {
         val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, eventId = eventId)
         val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/RoomEventSender.kt

@@ -1,75 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.session.room.send
-
-import androidx.work.BackoffPolicy
-import androidx.work.OneTimeWorkRequest
-import org.matrix.android.sdk.api.session.crypto.CryptoService
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.util.Cancelable
-import org.matrix.android.sdk.internal.di.SessionId
-import org.matrix.android.sdk.internal.di.WorkManagerProvider
-import org.matrix.android.sdk.internal.session.room.timeline.TimelineSendEventWorkCommon
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import org.matrix.android.sdk.internal.worker.startChain
-import timber.log.Timber
-import java.util.concurrent.TimeUnit
-import javax.inject.Inject
-
-internal class RoomEventSender @Inject constructor(
-        private val workManagerProvider: WorkManagerProvider,
-        private val timelineSendEventWorkCommon: TimelineSendEventWorkCommon,
-        @SessionId private val sessionId: String,
-        private val cryptoService: CryptoService
-) {
-    fun sendEvent(event: Event): Cancelable {
-        // Encrypted room handling
-        return if (cryptoService.isRoomEncrypted(event.roomId ?: "")
-                && !event.isEncrypted() // In case of resend where it's already encrypted so skip to send
-        ) {
-            Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule encrypt and send event ${event.eventId}")
-            val encryptWork = createEncryptEventWork(event, true)
-            // Note that event will be replaced by the result of the previous work
-            val sendWork = createSendEventWork(event, false)
-            timelineSendEventWorkCommon.postSequentialWorks(event.roomId ?: "", encryptWork, sendWork)
-        } else {
-            Timber.v("## SendEvent: [${System.currentTimeMillis()}] Schedule send event ${event.eventId}")
-            val sendWork = createSendEventWork(event, true)
-            timelineSendEventWorkCommon.postWork(event.roomId ?: "", sendWork)
-        }
-    }
-
-    private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        // Same parameter
-        val params = EncryptEventWorker.Params(sessionId, event.eventId!!)
-        val sendWorkData = WorkerParamsFactory.toData(params)
-
-        return workManagerProvider.matrixOneTimeWorkRequestBuilder<EncryptEventWorker>()
-                .setConstraints(WorkManagerProvider.workConstraints)
-                .setInputData(sendWorkData)
-                .startChain(startChain)
-                .setBackoffCriteria(BackoffPolicy.LINEAR, WorkManagerProvider.BACKOFF_DELAY, TimeUnit.MILLISECONDS)
-                .build()
-    }
-
-    private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest {
-        val sendContentWorkerParams = SendEventWorker.Params(sessionId = sessionId, eventId = event.eventId!!)
-        val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams)
-
-        return timelineSendEventWorkCommon.createWork<SendEventWorker>(sendWorkData, startChain)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/SendEventWorker.kt

@@ -22,12 +22,11 @@ import com.squareup.moshi.JsonClass
 import io.realm.RealmConfiguration
 import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.api.failure.shouldBeRetried
-import org.matrix.android.sdk.api.session.events.model.Content
+import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.internal.crypto.tasks.SendEventTask
 import org.matrix.android.sdk.internal.di.SessionDatabase
-import org.matrix.android.sdk.internal.network.executeRequest
 import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.session.room.RoomAPI
 import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
 import org.matrix.android.sdk.internal.worker.SessionWorkerParams
 import timber.log.Timber
@@ -47,11 +46,14 @@ internal class SendEventWorker(context: Context,
     internal data class Params(
             override val sessionId: String,
             override val lastFailureMessage: String? = null,
-            val eventId: String
+            val eventId: String,
+            // use this as an override if you want to send in clear in encrypted room
+            val isEncrypted: Boolean? = null
     ) : SessionWorkerParams
 
     @Inject lateinit var localEchoRepository: LocalEchoRepository
-    @Inject lateinit var roomAPI: RoomAPI
+    @Inject lateinit var sendEventTask: SendEventTask
+    @Inject lateinit var cryptoService: CryptoService
     @Inject lateinit var eventBus: EventBus
     @Inject lateinit var cancelSendTracker: CancelSendTracker
     @SessionDatabase @Inject lateinit var realmConfiguration: RealmConfiguration
@@ -63,7 +65,7 @@ internal class SendEventWorker(context: Context,
     override suspend fun doSafeWork(params: Params): Result {
         val event = localEchoRepository.getUpToDateEcho(params.eventId)
         if (event?.eventId == null || event.roomId == null) {
-            localEchoRepository.updateSendState(params.eventId, SendState.UNDELIVERED)
+            localEchoRepository.updateSendState(params.eventId, event?.roomId, SendState.UNDELIVERED)
             return Result.success()
                     .also { Timber.e("Work cancelled due to bad input data") }
         }
@@ -77,7 +79,7 @@ internal class SendEventWorker(context: Context,
         }
 
         if (params.lastFailureMessage != null) {
-            localEchoRepository.updateSendState(event.eventId, SendState.UNDELIVERED)
+            localEchoRepository.updateSendState(event.eventId, event.roomId, SendState.UNDELIVERED)
             // Transmit the error
             return Result.success(inputData)
                     .also { Timber.e("Work cancelled due to input error from parent") }
@@ -85,12 +87,12 @@ internal class SendEventWorker(context: Context,
 
         Timber.v("## SendEvent: [${System.currentTimeMillis()}] Send event ${params.eventId}")
         return try {
-            sendEvent(event.eventId, event.roomId, event.type, event.content)
+            sendEventTask.execute(SendEventTask.Params(event, params.isEncrypted ?: cryptoService.isRoomEncrypted(event.roomId)))
             Result.success()
         } catch (exception: Throwable) {
             if (/*currentAttemptCount >= MAX_NUMBER_OF_RETRY_BEFORE_FAILING ||**/ !exception.shouldBeRetried()) {
                 Timber.e("## SendEvent: [${System.currentTimeMillis()}]  Send event Failed cannot retry ${params.eventId} > ${exception.localizedMessage}")
-                localEchoRepository.updateSendState(event.eventId, SendState.UNDELIVERED)
+                localEchoRepository.updateSendState(event.eventId, event.roomId, SendState.UNDELIVERED)
                 return Result.success()
             } else {
                 Timber.e("## SendEvent: [${System.currentTimeMillis()}]  Send event Failed schedule retry ${params.eventId} > ${exception.localizedMessage}")
@@ -102,12 +104,4 @@ internal class SendEventWorker(context: Context,
     override fun buildErrorParams(params: Params, message: String): Params {
         return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
     }
-
-    private suspend fun sendEvent(eventId: String, roomId: String, type: String, content: Content?) {
-        localEchoRepository.updateSendState(eventId, SendState.SENDING)
-        executeRequest<SendResponse>(eventBus) {
-            apiCall = roomAPI.send(eventId, roomId, type, content)
-        }
-        localEchoRepository.updateSendState(eventId, SendState.SENT)
-    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/queue/EventSenderProcessor.kt

@@ -0,0 +1,239 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.send.queue
+
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import org.matrix.android.sdk.api.auth.data.SessionParams
+import org.matrix.android.sdk.api.auth.data.sessionId
+import org.matrix.android.sdk.api.extensions.tryOrNull
+import org.matrix.android.sdk.api.failure.Failure
+import org.matrix.android.sdk.api.failure.MatrixError
+import org.matrix.android.sdk.api.failure.isTokenError
+import org.matrix.android.sdk.api.session.crypto.CryptoService
+import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.api.session.sync.SyncState
+import org.matrix.android.sdk.api.util.Cancelable
+import org.matrix.android.sdk.internal.session.SessionScope
+import org.matrix.android.sdk.internal.task.TaskExecutor
+import timber.log.Timber
+import java.io.IOException
+import java.net.InetAddress
+import java.net.InetSocketAddress
+import java.net.Socket
+import java.util.Timer
+import java.util.TimerTask
+import java.util.concurrent.LinkedBlockingQueue
+import javax.inject.Inject
+import kotlin.concurrent.schedule
+
+/**
+ * A simple ever running thread unique for that session responsible of sending events in order.
+ * Each send is retried 3 times, if there is no network (e.g if cannot ping home server) it will wait and
+ * periodically test reachability before resume (does not count as a retry)
+ *
+ * If the app is killed before all event were sent, on next wakeup the scheduled events will be re posted
+ */
+@SessionScope
+internal class EventSenderProcessor @Inject constructor(
+        private val cryptoService: CryptoService,
+        private val sessionParams: SessionParams,
+        private val queuedTaskFactory: QueuedTaskFactory,
+        private val taskExecutor: TaskExecutor,
+        private val memento: QueueMemento
+) : Thread("SENDER_THREAD_SID_${sessionParams.credentials.sessionId()}") {
+
+    private fun markAsManaged(task: QueuedTask) {
+        memento.track(task)
+    }
+
+    private fun markAsFinished(task: QueuedTask) {
+        memento.unTrack(task)
+    }
+
+    // API
+    fun postEvent(event: Event): Cancelable {
+        return postEvent(event, event.roomId?.let { cryptoService.isRoomEncrypted(it) } ?: false)
+    }
+
+    override fun start() {
+        super.start()
+        // We should check for sending events not handled because app was killed
+        // But we should be careful of only took those that was submitted to us, because if it's
+        // for example it's a media event it is handled by some worker and he will handle it
+        // This is a bit fragile :/
+        // also some events cannot be retried manually by users, e.g reactions
+        // they were previously relying on workers to do the work :/ and was expected to always finally succeed
+        // Also some echos are not to be resent like redaction echos (fake event created for aggregation)
+
+        tryOrNull {
+            taskExecutor.executorScope.launch {
+                Timber.d("## Send relaunched pending events on restart")
+                memento.restoreTasks(this@EventSenderProcessor)
+            }
+        }
+    }
+
+    fun postEvent(event: Event, encrypt: Boolean): Cancelable {
+        val task = queuedTaskFactory.createSendTask(event, encrypt)
+        return postTask(task)
+    }
+
+    fun postRedaction(redactionLocalEcho: Event, reason: String?): Cancelable {
+        return postRedaction(redactionLocalEcho.eventId!!, redactionLocalEcho.redacts!!, redactionLocalEcho.roomId!!, reason)
+    }
+
+    fun postRedaction(redactionLocalEchoId: String, eventToRedactId: String, roomId: String, reason: String?): Cancelable {
+        val task = queuedTaskFactory.createRedactTask(redactionLocalEchoId, eventToRedactId, roomId, reason)
+        return postTask(task)
+    }
+
+    fun postTask(task: QueuedTask): Cancelable {
+        // non blocking add to queue
+        sendingQueue.add(task)
+        markAsManaged(task)
+        return object : Cancelable {
+            override fun cancel() {
+                task.cancel()
+            }
+        }
+    }
+
+    companion object {
+        private const val RETRY_WAIT_TIME_MS = 10_000L
+    }
+
+    private var sendingQueue = LinkedBlockingQueue<QueuedTask>()
+
+    private var networkAvailableLock = Object()
+    private var canReachServer = true
+    private var retryNoNetworkTask: TimerTask? = null
+
+    override fun run() {
+        Timber.v("## SendThread started ts:${System.currentTimeMillis()}")
+        try {
+            while (!isInterrupted) {
+                Timber.v("## SendThread wait for task to process")
+                val task = sendingQueue.take()
+                Timber.v("## SendThread Found task to process $task")
+
+                if (task.isCancelled()) {
+                    Timber.v("## SendThread send cancelled for $task")
+                    // we do not execute this one
+                    continue
+                }
+                // we check for network connectivity
+                while (!canReachServer) {
+                    Timber.v("## SendThread cannot reach server, wait ts:${System.currentTimeMillis()}")
+                    // schedule to retry
+                    waitForNetwork()
+                    // if thread as been killed meanwhile
+//                    if (state == State.KILLING) break
+                }
+                Timber.v("## Server is Reachable")
+                // so network is available
+
+                runBlocking {
+                    retryLoop@ while (task.retryCount < 3) {
+                        try {
+                            // SendPerformanceProfiler.startStage(task.event.eventId!!, SendPerformanceProfiler.Stages.SEND_WORKER)
+                            Timber.v("## SendThread retryLoop for $task retryCount ${task.retryCount}")
+                            task.execute()
+                            // sendEventTask.execute(SendEventTask.Params(task.event, task.encrypt, cryptoService))
+                            // SendPerformanceProfiler.stopStage(task.event.eventId, SendPerformanceProfiler.Stages.SEND_WORKER)
+                            break@retryLoop
+                        } catch (exception: Throwable) {
+                            when {
+                                exception is IOException || exception is Failure.NetworkConnection                         -> {
+                                    canReachServer = false
+                                    task.retryCount++
+                                    if (task.retryCount >= 3) task.onTaskFailed()
+                                    while (!canReachServer) {
+                                        Timber.v("## SendThread retryLoop cannot reach server, wait ts:${System.currentTimeMillis()}")
+                                        // schedule to retry
+                                        waitForNetwork()
+                                    }
+                                }
+                                (exception is Failure.ServerError && exception.error.code == MatrixError.M_LIMIT_EXCEEDED) -> {
+                                    task.retryCount++
+                                    if (task.retryCount >= 3) task.onTaskFailed()
+                                    Timber.v("## SendThread retryLoop retryable error for $task reason: ${exception.localizedMessage}")
+                                    // wait a bit
+                                    // Todo if its a quota exception can we get timout?
+                                    sleep(3_000)
+                                    continue@retryLoop
+                                }
+                                exception.isTokenError()                                                                   -> {
+                                    Timber.v("## SendThread retryLoop retryable TOKEN error, interrupt")
+                                    // we can exit the loop
+                                    task.onTaskFailed()
+                                    throw InterruptedException()
+                                }
+                                else                                                                                       -> {
+                                    Timber.v("## SendThread retryLoop Un-Retryable error, try next task")
+                                    // this task is in error, check next one?
+                                    break@retryLoop
+                                }
+                            }
+                        }
+                    }
+                }
+                markAsFinished(task)
+            }
+        } catch (interruptionException: InterruptedException) {
+            // will be thrown is thread is interrupted while seeping
+            interrupt()
+            Timber.v("## InterruptedException!! ${interruptionException.localizedMessage}")
+        }
+//        state = State.KILLED
+        // is this needed?
+        retryNoNetworkTask?.cancel()
+        Timber.w("## SendThread finished ${System.currentTimeMillis()}")
+    }
+
+    private fun waitForNetwork() {
+        retryNoNetworkTask = Timer(SyncState.NoNetwork.toString(), false).schedule(RETRY_WAIT_TIME_MS) {
+            synchronized(networkAvailableLock) {
+                canReachServer = checkHostAvailable().also {
+                    Timber.v("## SendThread checkHostAvailable $it")
+                }
+                networkAvailableLock.notify()
+            }
+        }
+        synchronized(networkAvailableLock) { networkAvailableLock.wait() }
+    }
+
+    /**
+     * Check if homeserver is reachable.
+     */
+    private fun checkHostAvailable(): Boolean {
+        val host = sessionParams.homeServerConnectionConfig.homeServerUri.host ?: return false
+        val port = sessionParams.homeServerConnectionConfig.homeServerUri.port.takeIf { it != -1 } ?: 80
+        val timeout = 30_000
+        try {
+            Socket().use { socket ->
+                val inetAddress: InetAddress = InetAddress.getByName(host)
+                val inetSocketAddress = InetSocketAddress(inetAddress, port)
+                socket.connect(inetSocketAddress, timeout)
+                return true
+            }
+        } catch (e: IOException) {
+            Timber.v("## EventSender isHostAvailable failure ${e.localizedMessage}")
+            return false
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/queue/QueueMemento.kt

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.session.room.send.queue
+
+import android.content.Context
+import org.matrix.android.sdk.api.auth.data.sessionId
+import org.matrix.android.sdk.api.extensions.tryOrNull
+import org.matrix.android.sdk.api.session.crypto.CryptoService
+import org.matrix.android.sdk.api.session.room.send.SendState
+import org.matrix.android.sdk.internal.di.SessionId
+import org.matrix.android.sdk.internal.session.room.send.LocalEchoRepository
+import timber.log.Timber
+import javax.inject.Inject
+
+/**
+ * Simple lightweight persistence
+ * Don't want to go in DB due to current issues
+ * Will never manage lots of events, it simply uses sharedPreferences.
+ * It is just used to remember what events/localEchos was managed by the event sender in order to
+ * reschedule them (and only them) on next restart
+ */
+internal class QueueMemento @Inject constructor(context: Context,
+                                                @SessionId sessionId: String,
+                                                private val queuedTaskFactory: QueuedTaskFactory,
+                                                private val localEchoRepository: LocalEchoRepository,
+                                                private val cryptoService: CryptoService) {
+
+    private val storage = context.getSharedPreferences("QueueMemento_$sessionId", Context.MODE_PRIVATE)
+    private val managedTaskInfos = mutableListOf<QueuedTask>()
+
+    fun track(task: QueuedTask) {
+        synchronized(managedTaskInfos) {
+            managedTaskInfos.add(task)
+            persist()
+        }
+    }
+
+    fun unTrack(task: QueuedTask) {
+        managedTaskInfos.remove(task)
+        persist()
+    }
+
+    private fun persist() {
+        managedTaskInfos.mapIndexedNotNull { index, queuedTask ->
+            toTaskInfo(queuedTask, index)?.let { TaskInfo.map(it) }
+        }.toSet().let { set ->
+            storage.edit()
+                    .putStringSet("ManagedBySender", set)
+                    .apply()
+        }
+    }
+
+    private fun toTaskInfo(task: QueuedTask, order: Int): TaskInfo? {
+        synchronized(managedTaskInfos) {
+            return when (task) {
+                is SendEventQueuedTask -> SendEventTaskInfo(
+                        localEchoId = task.event.eventId ?: "",
+                        encrypt = task.encrypt,
+                        order = order
+                )
+                is RedactQueuedTask    -> RedactEventTaskInfo(
+                        redactionLocalEcho = task.redactionLocalEchoId,
+                        order = order
+                )
+                else                   -> null
+            }
+        }
+    }
+
+    suspend fun restoreTasks(eventProcessor: EventSenderProcessor) {
+        // events should be restarted in correct order
+        storage.getStringSet("ManagedBySender", null)?.let { pending ->
+            Timber.d("## Send - Recovering unsent events $pending")
+            pending.mapNotNull { tryOrNull { TaskInfo.map(it) } }
+        }
+                ?.sortedBy { it.order }
+                ?.forEach { info ->
+                    try {
+                        when (info) {
+                            is SendEventTaskInfo   -> {
+                                localEchoRepository.getUpToDateEcho(info.localEchoId)?.let {
+                                    if (it.sendState.isSending() && it.eventId != null && it.roomId != null) {
+                                        localEchoRepository.updateSendState(it.eventId, it.roomId, SendState.UNSENT)
+                                        Timber.d("## Send -Reschedule send $info")
+                                        eventProcessor.postTask(queuedTaskFactory.createSendTask(it, info.encrypt ?: cryptoService.isRoomEncrypted(it.roomId)))
+                                    }
+                                }
+                            }
+                            is RedactEventTaskInfo -> {
+                                info.redactionLocalEcho?.let { localEchoRepository.getUpToDateEcho(it) }?.let {
+                                    localEchoRepository.updateSendState(it.eventId!!, it.roomId, SendState.UNSENT)
+                                    // try to get reason
+                                    val reason = it.content?.get("reason") as? String
+                                    if (it.redacts != null && it.roomId != null) {
+                                        Timber.d("## Send -Reschedule redact $info")
+                                        eventProcessor.postTask(queuedTaskFactory.createRedactTask(it.eventId, it.redacts, it.roomId, reason))
+                                    }
+                                }
+                                // postTask(queuedTaskFactory.createRedactTask(info.eventToRedactId, info.)
+                            }
+                        }
+                    } catch (failure: Throwable) {
+                        Timber.e("failed to restore task $info")
+                    }
+                }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/send/queue/QueuedTask.kt

@@ -13,14 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.matrix.android.sdk.internal.crypto.crosssigning
 
-data class SessionToCryptoRoomMembersUpdate(
+package org.matrix.android.sdk.internal.session.room.send.queue
-        val roomId: String,
-        val isDirect: Boolean,
-        val userIds: List<String>
-)
 
-data class CryptoToSessionUserTrustChange(
+abstract class QueuedTask {
-        val userIds: List<String>
+    var retryCount = 0
-)
+
+    abstract suspend fun execute()
+
+    abstract fun onTaskFailed()
+
+    abstract fun isCancelled() : Boolean
+
+    abstract fun cancel()
+}