Merge branch 'feature/eric/improve-back-navigation' into experiment/eric/space-switching-modal

# Conflicts:
#	vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt
#	vector/src/main/res/layout/fragment_home_detail.xml

.github/ISSUE_TEMPLATE/release.yml

@@ -23,7 +23,8 @@ body:
 
         ### Do the release
 
-        - [ ] Create release with gitflow, branch name `release/1.2.3`
+        - [ ] Make sure `develop` and `main` are up to date (git pull)
+        - [ ] Checkout develop and create a release with gitflow, branch name `release/1.2.3`
         - [ ] Check the crashes from the PlayStore
         - [ ] Check the rageshake with the current dev version: https://github.com/matrix-org/element-android-rageshakes/labels/1.2.3-dev
         - [ ] Run the integration test, and especially `UiAllScreensSanityTest.allScreensTest()`

.github/dependabot.yml

@@ -10,6 +10,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    reviewers:
+      - "vector-im/element-android-reviewers"
     ignore:
       - dependency-name: "*github-script*"
   # Updates for Gradle dependencies used in the app
@@ -19,6 +21,6 @@ updates:
       interval: "daily"
     open-pull-requests-limit: 200
     reviewers:
-      - "bmarty"
+      - "vector-im/element-android-reviewers"
     ignore:
       - dependency-name: com.google.zxing:core

.github/workflows/build.yml

@@ -67,4 +67,4 @@ jobs:
           path: |
             vector/build/outputs/apk/*/release/*.apk
 
-# TODO: add exodus checks
+# TODO add exodus checks

.github/workflows/docs.yml

@@ -0,0 +1,22 @@
+name: Documentation
+
+on:
+  push:
+    branches: [ develop ]
+
+jobs:
+  docs:
+    name: Generate and publish Android Matrix SDK documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Build docs
+        run: ./gradlew dokkaHtml
+
+      - name: Deploy docs
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./matrix-sdk-android/build/dokka/html

.github/workflows/post-pr.yml

@@ -1,28 +1,43 @@
-name: Nightly Tests
+name: Integration Tests
+
+# This runs for all closed pull requests against main, including those closed without merge.
+# Further filtering occurs in 'should-i-run'
 
 on:
-  push:
+  pull_request:
-    branches: [ release/* ]
+    types: [closed]
-  schedule:
+    branches: [develop]
-    # At 20:00 every day UTC
-    - cron: '0 20 * * *'
-  workflow_dispatch:
 
 # Enrich gradle.properties for CI/CD
 env:
   CI_GRADLE_ARG_PROPERTIES: >
     -Porg.gradle.jvmargs=-Xmx4g
     -Porg.gradle.parallel=false
+
 jobs:
+
+  # More info on should-i-run:
+  # If this fails to run (the IF doesn't complete) then the needs will not be satisfied for any of the 
+  # other jobs below, so none will run.
+  # except for the notification job at the bottom which will run all the time, unless should-i-run isn't
+  # successful, or all the other jobs have succeeded
+
+  should-i-run:
+    name: Check if PR is suitable for analysis
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged # Additionally require PR to have been completely merged.
+    steps:
+      - run: echo "Run those tests!" # no-op success
+        
   # Run Android Tests
   integration-tests:
     name: Matrix SDK - Running Integration Tests
+    needs: should-i-run
     runs-on: macos-latest
     strategy:
       fail-fast: false
       matrix:
         api-level: [ 28 ]
-    # No concurrency required, runs every time on a schedule.
     steps:
       - uses: actions/checkout@v3
       - uses: gradle/wrapper-validation-action@v1
@@ -43,11 +58,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gradle-
       - name: Start synapse server
-        uses: michaelkaye/setup-matrix-synapse@v1.0.1
+        uses: michaelkaye/setup-matrix-synapse@v1.0.3
         with:
           uploadLogs: true
           httpPort: 8080
           disableRateLimiting: true
+          public_baseurl: "http://10.0.2.2:8080/"
       # package: org.matrix.android.sdk.session
       - name: Run integration tests for  Matrix SDK [org.matrix.android.sdk.session] API[${{ matrix.api-level }}]
         uses: reactivecircus/android-emulator-runner@v2
@@ -209,6 +225,7 @@ jobs:
 
   ui-tests:
     name: UI Tests (Synapse)
+    needs: should-i-run
     runs-on: macos-latest
     strategy:
       fail-fast: false
@@ -230,11 +247,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gradle-
       - name: Start synapse server
-        uses: michaelkaye/setup-matrix-synapse@v1.0.1
+        uses: michaelkaye/setup-matrix-synapse@v1.0.3
         with:
           uploadLogs: true
           httpPort: 8080
           disableRateLimiting: true
+          public_baseurl: "http://10.0.2.2:8080/"
       - uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
@@ -266,7 +284,8 @@ jobs:
 
   codecov-units:
     name: Unit tests with code coverage
-    runs-on: macos-latest
+    needs: should-i-run
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
@@ -290,49 +309,21 @@ jobs:
           path: |
             build/reports/jacoco/allCodeCoverageReport/allCodeCoverageReport.xml
 
-  sonarqube:
+# Notify the channel about delayed failures
-    name: Sonarqube upload
-    runs-on: macos-latest
-    if: always() && github.event_name == 'schedule'
-    needs:
-      - codecov-units
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: 'adopt'
-          java-version: '11'
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
-      - uses: actions/download-artifact@v3
-        with:
-          name: codecov-xml # will restore to allCodeCoverageReport.xml by default; we restore to the same location in following tasks
-      - run: mkdir -p build/reports/jacoco/allCodeCoverageReport/
-      - run: mv allCodeCoverageReport.xml build/reports/jacoco/allCodeCoverageReport/
-      - run: ./gradlew sonarqube $CI_GRADLE_ARG_PROPERTIES
-        env:
-          ORG_GRADLE_PROJECT_SONAR_LOGIN: ${{ secrets.SONAR_TOKEN }}
-
-# Notify the channel about scheduled runs, or pushes to the release branches, do not notify for manually triggered runs
   notify:
     name: Notify matrix
     runs-on: ubuntu-latest
     needs:
+      - should-i-run
       - integration-tests
       - ui-tests
-      - sonarqube
+      - codecov-units
-    if: always() && github.event_name != 'workflow_dispatch'
+    if: always() && (needs.should-i-run.result == 'success' ) && ((needs.codecov-units.result != 'success' ) || (needs.ui-tests.result != 'success') || (needs.integration-tests.result != 'success'))
     # No concurrency required, runs every time on a schedule.
     steps:
       - uses: michaelkaye/matrix-hookshot-action@v1.0.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           hookshot_url: ${{ secrets.ELEMENT_ANDROID_HOOKSHOT_URL }}
-          text_template: "{{#if '${{ github.event_name }}' == 'schedule' }}Nightly test run{{else}}Test run (on ${{ github.ref }}){{/if }}: {{#each job_statuses }}{{#with this }}{{#if completed }}  {{name}} {{conclusion}} at {{completed_at}}, {{/if}}{{/with}}{{/each}}"
+          text_template: "Post-merge validation of ${{ github.head_ref }} into ${{ github.base_ref }} by ${{ github.event.pull_request.merged_by }} failed: {{#each job_statuses }}{{#with this }}{{#if completed }}  {{name}} {{conclusion}} at {{completed_at}}, {{/if}}{{/with}}{{/each}}"
-          html_template: "{{#if '${{ github.event_name }}' == 'schedule' }}Nightly test run{{else}}Test run (on ${{ github.ref }}){{/if }}: {{#each job_statuses }}{{#with this }}{{#if completed }}<br />{{icon conclusion}} {{name}} <font color='{{color conclusion}}'>{{conclusion}} at {{completed_at}} <a href=\"{{html_url}}\">[details]</a></font>{{/if}}{{/with}}{{/each}}"
+          html_template: "Post-merge validation of ${{ github.head_ref }} into ${{ github.base_ref }} by ${{ github.event.pull_request.merged_by }} failed: {{#each job_statuses }}{{#with this }}{{#if completed }}<br />{{icon conclusion}} {{name}} <font color='{{color conclusion}}'>{{conclusion}} at {{completed_at}} <a href=\"{{html_url}}\">[details]</a></font>{{/if}}{{/with}}{{/each}}"

.github/workflows/quality.yml

@@ -147,3 +147,23 @@ jobs:
           name: release-lint-report-${{ matrix.target }}
           path: |
             vector/build/reports/*.*
+
+  detekt:
+    name: Detekt Analysis
+    runs-on: ubuntu-latest
+    # Allow all jobs on main and develop. Just one per PR.
+    concurrency:
+      group: ${{ github.ref == 'refs/heads/main' && format('detekt-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('detekt-develop-{0}', github.sha) || format('detekt-{0}', github.ref) }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run detekt
+        run: |
+          ./gradlew detekt
+      - name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: detekt-report
+          path: |
+            */build/reports/detekt/detekt.html

.github/workflows/sonarqube.yml

@@ -0,0 +1,81 @@
+name: Sonarqube nightly
+
+on:
+  schedule:
+    - cron: '0 20 * * *'
+
+# Enrich gradle.properties for CI/CD
+env:
+  CI_GRADLE_ARG_PROPERTIES: >
+    -Porg.gradle.jvmargs=-Xmx4g
+    -Porg.gradle.parallel=false
+jobs:
+  codecov-units:
+    name: Unit tests with code coverage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with: 
+          distribution: 'adopt'
+          java-version: '11'
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - run: ./gradlew allCodeCoverageReport $CI_GRADLE_ARG_PROPERTIES
+      - name: Upload Codecov data
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: codecov-xml
+          path: |
+            build/reports/jacoco/allCodeCoverageReport/allCodeCoverageReport.xml
+
+  sonarqube:
+    name: Sonarqube upload
+    runs-on: ubuntu-latest
+    needs:
+      - codecov-units
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - uses: actions/download-artifact@v3
+        with:
+          name: codecov-xml # will restore to allCodeCoverageReport.xml by default; we restore to the same location in following tasks
+      - run: mkdir -p build/reports/jacoco/allCodeCoverageReport/
+      - run: mv allCodeCoverageReport.xml build/reports/jacoco/allCodeCoverageReport/
+      - run: ./gradlew sonarqube $CI_GRADLE_ARG_PROPERTIES
+        env:
+          ORG_GRADLE_PROJECT_SONAR_LOGIN: ${{ secrets.SONAR_TOKEN }}
+
+# Notify the channel about sonarqube failures
+  notify:
+    name: Notify matrix
+    runs-on: ubuntu-latest
+    needs:
+      - sonarqube
+      - codecov-units
+    if: always() && (needs.sonarqube.result != 'success' || needs.codecov-units.result != 'success') 
+    steps:
+      - uses: michaelkaye/matrix-hookshot-action@v1.0.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          hookshot_url: ${{ secrets.ELEMENT_ANDROID_HOOKSHOT_URL }}
+          text_template: "Sonarqube run (on ${{ github.ref }}): {{#each job_statuses }}{{#with this }}{{#if completed }}  {{name}} {{conclusion}} at {{completed_at}}, {{/if}}{{/with}}{{/each}}"
+          html_template: "Sonarqube run (on ${{ github.ref }}): {{#each job_statuses }}{{#with this }}{{#if completed }}<br />{{icon conclusion}} {{name}} <font color='{{color conclusion}}'>{{conclusion}} at {{completed_at}} <a href=\"{{html_url}}\">[details]</a></font>{{/if}}{{/with}}{{/each}}"

README.md

@@ -14,7 +14,7 @@ It is a total rewrite of [Riot-Android](https://github.com/vector-im/riot-androi
 [<img src="resources/img/google-play-badge.png" alt="Get it on Google Play" height="60">](https://play.google.com/store/apps/details?id=im.vector.app)
 [<img src="resources/img/f-droid-badge.png" alt="Get it on F-Droid" height="60">](https://f-droid.org/app/im.vector.app)
 
-Nightly build: [![Buildkite](https://badge.buildkite.com/ad0065c1b70f557cd3b1d3d68f9c2154010f83c4d6f71706a9.svg?branch=develop)](https://buildkite.com/matrix-dot-org/element-android/builds?branch=develop) Nighly test status: [![allScreensTest](https://github.com/vector-im/element-android/actions/workflows/nightly.yml/badge.svg)](https://github.com/vector-im/element-android/actions/workflows/nightly.yml)
+Nightly build: [![Buildkite](https://badge.buildkite.com/ad0065c1b70f557cd3b1d3d68f9c2154010f83c4d6f71706a9.svg?branch=develop)](https://buildkite.com/matrix-dot-org/element-android/builds?branch=develop) Nightly test status: [![allScreensTest](https://github.com/vector-im/element-android/actions/workflows/nightly.yml/badge.svg)](https://github.com/vector-im/element-android/actions/workflows/nightly.yml)
 
 
 # New Android SDK
@@ -53,3 +53,4 @@ Come chat with the community in the dedicated Matrix [room](https://matrix.to/#/
 Issues are triaged by community members and the Android App Team, following the [triage process](https://github.com/vector-im/element-meta/wiki/Triage-process).
 
 We use [issue labels](https://github.com/vector-im/element-meta/wiki/Issue-labelling) to sort all incoming issues.
+

build.gradle

@@ -5,10 +5,17 @@ buildscript {
     apply from: 'dependencies_groups.gradle'
 
     repositories {
-        google()
+        // Do not use `google()`, it prevents Dependabot from working properly
+        maven {
+            url 'https://maven.google.com'
+        }
         maven {
             url "https://plugins.gradle.org/m2/"
         }
+        // Do not use `mavenCentral()`, it prevents Dependabot from working properly
+        maven {
+            url 'https://repo1.maven.org/maven2'
+        }
     }
 
     dependencies {
@@ -28,9 +35,11 @@ buildscript {
     }
 }
 
-// ktlint Plugin
 plugins {
+    // ktlint Plugin
     id "org.jlleitschuh.gradle.ktlint" version "10.3.0"
+    // Detekt
+    id "io.gitlab.arturbosch.detekt" version "1.20.0"
 }
 
 // https://github.com/jeremylong/DependencyCheck
@@ -45,9 +54,12 @@ dependencyCheck {
 
 allprojects {
     apply plugin: "org.jlleitschuh.gradle.ktlint"
+    apply plugin: "io.gitlab.arturbosch.detekt"
 
     repositories {
-        mavenCentral {
+        // Do not use `mavenCentral()`, it prevents Dependabot from working properly
+        maven {
+            url 'https://repo1.maven.org/maven2'
             content {
                 groups.mavenCentral.regex.each { includeGroupByRegex it }
                 groups.mavenCentral.group.each { includeGroup it }
@@ -70,14 +82,18 @@ allprojects {
                 groups.jitsi.group.each { includeGroup it }
             }
         }
-        google {
+        // Do not use `google()`, it prevents Dependabot from working properly
+        maven {
+            url 'https://maven.google.com'
             content {
                 groups.google.regex.each { includeGroupByRegex it }
                 groups.google.group.each { includeGroup it }
             }
         }
         //noinspection JcenterRepositoryObsolete
-        jcenter {
+        // Do not use `jcenter`, it prevents Dependabot from working properly
+        maven {
+            url 'https://jcenter.bintray.com'
             content {
                 groups.jcenter.regex.each { includeGroupByRegex it }
                 groups.jcenter.group.each { includeGroup it }
@@ -106,7 +122,7 @@ allprojects {
         // display the corresponding rule
         verbose = true
         disabledRules = [
-                // TODO: Re-enable these 4 rules after reformatting project
+                // TODO Re-enable these 4 rules after reformatting project
                 "indent",
                 "experimental:argument-list-wrapping",
                 "max-line-length",
@@ -127,6 +143,15 @@ allprojects {
                 "experimental:kdoc-wrapping",
         ]
     }
+
+    detekt {
+        // preconfigure defaults
+        buildUponDefaultConfig = true
+        // activate all available (even unstable) rules.
+        allRules = true
+        // point to your custom config defining rules to run, overwriting default behavior
+        config = files("$rootDir/tools/detekt/detekt.yml")
+    }
 }
 
 task clean(type: Delete) {

changelog.d/5151.misc

@@ -0,0 +1 @@
+Improve threads rendering in the main timeline

changelog.d/5494.feature

@@ -0,0 +1 @@
+Use key backup before requesting keys + refactor & improvement of key request/forward

changelog.d/5559.sdk

@@ -0,0 +1,4 @@
+- New API to enable/disable key forwarding CryptoService#enableKeyGossiping()
+- New API to limit room key request only to own devices MXCryptoConfig#limitRoomKeyRequestsToMyDevices
+- Event Trail API has changed, now using AuditTrail events
+- New API to manually accept an incoming key request CryptoService#manuallyAcceptRoomKeyRequest()

changelog.d/5825.bugfix

@@ -0,0 +1 @@
+Changed copy and list order in member profile screen.

changelog.d/5906.bugfix

@@ -0,0 +1 @@
+Desynchronized 4S | Megolm backup causing Unusable backup

changelog.d/5911.feature

@@ -0,0 +1 @@
+Screen sharing over WebRTC

changelog.d/5932.feature

@@ -0,0 +1 @@
+Allow using the latest user Avatar and name for all messages in the timeline

changelog.d/5936.feature

@@ -0,0 +1 @@
+Added themed launch icons for Android 13

changelog.d/5941.bugfix

@@ -0,0 +1 @@
+If animations are disable on the System, chat effects and confetti will be disabled too

changelog.d/5965.sdk

@@ -0,0 +1 @@
+Including SSL/TLS error handing when doing WellKnown lookups without a custom HomeServerConnectionConfig

changelog.d/5973.doc

@@ -0,0 +1 @@
+Note public_baseurl requirement in integration tests documentation.

changelog.d/5997.misc

@@ -0,0 +1 @@
+Update check for server-side threads support to match spec.

changelog.d/6038.misc

@@ -0,0 +1 @@
+Setup detekt

changelog.d/6047.feature

@@ -0,0 +1 @@
+Add presence indicator busy and away.

changelog.d/6073.feature

@@ -0,0 +1 @@
+Adds up navigation in spaces

dependencies.gradle

@@ -7,26 +7,26 @@ ext.versions = [
         'targetCompat'      : JavaVersion.VERSION_11,
 ]
 
-def gradle = "7.0.4"
+def gradle = "7.2.0"
 // Ref: https://kotlinlang.org/releases.html
-def kotlin = "1.6.0"
+def kotlin = "1.6.21"
-def kotlinCoroutines = "1.6.0"
+def kotlinCoroutines = "1.6.1"
-def dagger = "2.40.5"
+def dagger = "2.42"
 def retrofit = "2.9.0"
 def arrow = "0.8.2"
 def markwon = "4.6.2"
 def moshi = "1.13.0"
-def lifecycle = "2.4.0"
+def lifecycle = "2.4.1"
 def flowBinding = "1.2.0"
 def epoxy = "4.6.2"
-def mavericks = "2.5.0"
+def mavericks = "2.6.1"
-def glide = "4.12.0"
+def glide = "4.13.2"
 def bigImageViewer = "1.8.1"
-def jjwt = "0.11.2"
+def jjwt = "0.11.5"
-def vanniktechEmoji = "0.8.0"
+def vanniktechEmoji = "0.9.0"
 
 // Testing
-def mockk = "1.12.1"
+def mockk = "1.12.4"
 def espresso = "3.4.0"
 def androidxTest = "1.4.0"
 def androidxOrchestrator = "1.4.1"
@@ -45,15 +45,15 @@ ext.libs = [
                 'coroutinesTest'          : "org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinCoroutines"
         ],
         androidx    : [
-                'appCompat'               : "androidx.appcompat:appcompat:1.4.0",
+                'appCompat'               : "androidx.appcompat:appcompat:1.4.1",
                 'core'                    : "androidx.core:core-ktx:1.7.0",
                 'recyclerview'            : "androidx.recyclerview:recyclerview:1.2.1",
                 'exifinterface'           : "androidx.exifinterface:exifinterface:1.3.3",
-                'fragmentKtx'             : "androidx.fragment:fragment-ktx:1.4.0",
+                'fragmentKtx'             : "androidx.fragment:fragment-ktx:1.4.1",
-                'constraintLayout'        : "androidx.constraintlayout:constraintlayout:2.1.2",
+                'constraintLayout'        : "androidx.constraintlayout:constraintlayout:2.1.3",
                 'work'                    : "androidx.work:work-runtime-ktx:2.7.1",
                 'autoFill'                : "androidx.autofill:autofill:1.1.0",
-                'preferenceKtx'           : "androidx.preference:preference-ktx:1.1.1",
+                'preferenceKtx'           : "androidx.preference:preference-ktx:1.2.0",
                 'junit'                   : "androidx.test.ext:junit:1.1.3",
                 'lifecycleCommon'         : "androidx.lifecycle:lifecycle-common:$lifecycle",
                 'lifecycleLivedata'       : "androidx.lifecycle:lifecycle-livedata-ktx:$lifecycle",
@@ -72,7 +72,7 @@ ext.libs = [
                 'espressoIntents'         : "androidx.test.espresso:espresso-intents:$espresso"
         ],
         google      : [
-                'material'                : "com.google.android.material:material:1.5.0"
+                'material'                : "com.google.android.material:material:1.6.0"
         ],
         dagger      : [
                 'dagger'                  : "com.google.dagger:dagger:$dagger",

dependencies_groups.gradle

@@ -123,6 +123,7 @@ ext.groups = [
                         'io.github.detekt.sarif4k',
                         'io.github.microutils',
                         'io.github.reactivecircus.flowbinding',
+                        'io.gitlab.arturbosch.detekt',
                         'io.grpc',
                         'io.jsonwebtoken',
                         'io.kindedj',
@@ -195,6 +196,7 @@ ext.groups = [
                         'org.testng',
                         'org.threeten',
                         'org.webjars',
+                        'org.yaml',
                         'ru.noties',
                         'xerces',
                         'xml-apis',

docs/integration_tests.md

@@ -43,14 +43,17 @@ virtualenv -p python3 env
 source env/bin/activate
 pip install -e .
 demo/start.sh --no-rate-limit
+
 ```
 
-Alternatively, to install the latest Synapse release package (and not a cloned branch) you can run the following instead of `pip install -e .`:
+Alternatively, to install the latest Synapse release package (and not a cloned branch) you can run the following instead of `git clone` and `pip install -e .`:
 
 ```bash
 pip install matrix-synapse
 ```
 
+On your first run, you will want to stop the demo and edit the config to correct the `public_baseurl` to http://10.0.2.2:8080 and restart the server.
+
 You should now have 3 running federated Synapse instances 🎉, at http://127.0.0.1:8080/, http://127.0.0.1:8081/ and  http://127.0.0.1:8082/, which should display a "It Works! Synapse is running" message.
 
 ## Run the test
@@ -87,6 +90,18 @@ You'll need python3 to be able to run synapse
 
 Try on the Emulator browser to open "http://10.0.2.2:8080". You should see the "Synapse is running" message.
 
+### Tests partially run but some fail with "Unable to contact localhost:8080"
+
+This is because the `public_baseurl` of synapse is not consistent with the endpoint that the tests are connecting to.
+
+Ensure you have the following configuration in `demo/etc/8080.config`.
+
+```
+public_baseurl: http://10.0.2.2:8080/
+```
+
+After changing this you will need to restart synapse using `demo/stop.sh` and `demo/start.sh` to load the new configuration.
+
 ### virtualenv command fails
 
 You can try using

docs/pull_request.md

@@ -30,6 +30,19 @@ In any case, it is better to explicitly declare in the description why the PR is
 
 Also, draft PR should not stay indefinitely in this state. It may be removed if it is the case and the submitter does not update it after a few days.
 
+##### Base branch
+
+The `develop` branch is generally the base branch for every PRs.
+
+Exceptions can occur:
+
+- if a feature implementation is split into multiple PRs. We can have a chain of PRs in this case. PR can be merged one by one on develop, and GitHub change the target branch to `develop` for the next PR automatically.
+- we want to merge a PR from the community, but there is still work to do, and the PR is not updated by the submitter. First, we can kindly ask the submitter if they will update their PR, by commenting it. If there is no answer after a few days (including a week-end), we can create a new branch, push it, and change the target branch of the PR to this new branch. The PR can then be merged, and we can add more commits to fix the issues. After that a new PR can be created with `develop` as a target branch.
+
+**Important notice 1:** Releases are created from the `develop` branch. So `develop` branch should always contain a "releasable" source code. So when a feature is being implemented with several PRs, it has to be disabled by default (using a feature flag for instance), until the feature is fully implemented. A last PR to enable the feature can then be created.
+
+**Important notice 2:** Database migration: some developers and some people from the community are using the nightly build from `develop`. Multiple database migrations should be properly handled for them. This is OK to have multiple migrations between 2 releases, this is not OK to add steps to the pending database migration on `develop`. So for instance `develop` users will migrate from version 11 to version 12, then 13, then 14, and `main` users will do all those steps after they get the app upgrade.
+
 ##### PR Review Assignment
 
 We use automatic assignment for PR reviews. A PR is automatically routed by GitHub to 2 team members using the round robin algorithm. The process is the following:

docs/ui-tests.md

@@ -176,4 +176,4 @@ class SettingsAdvancedRobot {
         clickOn(R.string.settings_developer_mode_summary)
     }
 }
-```
+```

library/core-utils/build.gradle

@@ -44,7 +44,7 @@ android {
     kotlinOptions {
         jvmTarget = "11"
         freeCompilerArgs += [
-                "-Xopt-in=kotlin.RequiresOptIn"
+                "-opt-in=kotlin.RequiresOptIn"
         ]
     }
 }
@@ -52,4 +52,4 @@ android {
 dependencies {
     implementation libs.androidx.appCompat
     implementation libs.jetbrains.coroutinesAndroid
-}
+}

library/jsonviewer/build.gradle

@@ -6,8 +6,14 @@ apply plugin: 'com.jakewharton.butterknife'
 
 buildscript {
     repositories {
-        google()
+        // Do not use `google()`, it prevents Dependabot from working properly
-        mavenCentral()
+        maven {
+            url 'https://maven.google.com'
+        }
+        // Do not use `mavenCentral()`, it prevents Dependabot from working properly
+        maven {
+            url 'https://repo1.maven.org/maven2'
+        }
     }
     dependencies {
         classpath 'com.jakewharton:butterknife-gradle-plugin:10.2.3'

library/multipicker/src/main/java/im/vector/lib/multipicker/MultiPicker.kt

@@ -16,7 +16,7 @@
 
 package im.vector.lib.multipicker
 
-class MultiPicker<T> {
+class MultiPicker<T> private constructor() {
 
     companion object Type {
         val IMAGE by lazy { MultiPicker<ImagePicker>() }

library/ui-styles/src/main/res/values/colors.xml

@@ -128,6 +128,14 @@
     <color name="vctr_presence_indicator_online_light">@color/palette_element_green</color>
     <color name="vctr_presence_indicator_online_dark">@color/palette_element_green</color>
 
+    <attr name="vctr_presence_indicator_busy" format="color" />
+    <color name="vctr_presence_indicator_busy_light">@color/element_alert_light</color>
+    <color name="vctr_presence_indicator_busy_dark">@color/element_alert_dark</color>
+
+    <attr name="vctr_presence_indicator_away" format="color" />
+    <color name="vctr_presence_indicator_away_light">@color/palette_element_orange</color>
+    <color name="vctr_presence_indicator_away_dark">@color/palette_element_orange</color>
+
     <!-- Location sharing colors -->
     <attr name="vctr_live_location" format="color" />
     <color name="vctr_live_location_light">@color/palette_prune</color>

library/ui-styles/src/main/res/values/palette.xml

@@ -17,6 +17,7 @@
     <color name="palette_melon">#FF812D</color>
 
     <color name="palette_element_green">#0DBD8B</color>
+    <color name="palette_element_orange">#D9B072</color>
     <color name="palette_white">#FFFFFF</color>
     <color name="palette_vermilion">#FF5B55</color>
     <!-- (unused) -->

library/ui-styles/src/main/res/values/theme_dark.xml

@@ -44,6 +44,8 @@
         <!-- Presence Indicator colors -->
         <item name="vctr_presence_indicator_offline">@color/vctr_presence_indicator_offline_dark</item>
         <item name="vctr_presence_indicator_online">@color/vctr_presence_indicator_online_dark</item>
+        <item name="vctr_presence_indicator_busy">@color/vctr_presence_indicator_busy_dark</item>
+        <item name="vctr_presence_indicator_away">@color/vctr_presence_indicator_away_dark</item>
 
         <!-- Some aliases -->
         <item name="vctr_header_background">?vctr_system</item>

library/ui-styles/src/main/res/values/theme_light.xml

@@ -44,6 +44,8 @@
         <!-- Presence Indicator colors -->
         <item name="vctr_presence_indicator_offline">@color/vctr_presence_indicator_offline_light</item>
         <item name="vctr_presence_indicator_online">@color/vctr_presence_indicator_online_light</item>
+        <item name="vctr_presence_indicator_busy">@color/vctr_presence_indicator_busy_light</item>
+        <item name="vctr_presence_indicator_away">@color/vctr_presence_indicator_away_light</item>
 
         <!-- Some aliases -->
         <item name="vctr_header_background">?vctr_system</item>

matrix-sdk-android/build.gradle

@@ -7,7 +7,10 @@ apply plugin: "org.jetbrains.dokka"
 
 buildscript {
     repositories {
-        mavenCentral()
+        // Do not use `mavenCentral()`, it prevents Dependabot from working properly
+        maven {
+            url 'https://repo1.maven.org/maven2'
+        }
     }
     dependencies {
         classpath "io.realm:realm-gradle-plugin:10.9.0"
@@ -98,6 +101,9 @@ android {
         freeCompilerArgs += [
                 // Disabled for now, there are too many errors. Could be handled in another dedicated PR
                 // '-Xexplicit-api=strict', // or warning
+                "-opt-in=kotlin.RequiresOptIn",
+                // Opt in for kotlinx.coroutines.FlowPreview
+                "-opt-in=kotlinx.coroutines.FlowPreview",
         ]
     }
 
@@ -175,7 +181,7 @@ dependencies {
     implementation libs.arrow.instances
 
     // olm lib is now hosted in MavenCentral
-    implementation 'org.matrix.android:olm-sdk:3.2.10'
+    implementation 'org.matrix.android:olm-sdk:3.2.11'
 
     // DI
     implementation libs.dagger.dagger

matrix-sdk-android/docs/modules.md

@@ -1,5 +1,8 @@
 # Module matrix-sdk-android
 
+<!-- Note: the line below will appear only when the documentation is generated from Element-Android project, and not when it's generated from the SDK project -->
+**Note**: You are viewing the nightly documentation of the Android Matrix SDK library. The documentation of the released library can be found here: [https://matrix-org.github.io/matrix-android-sdk2/](https://matrix-org.github.io/matrix-android-sdk2/)
+
 ## Welcome to the matrix-sdk-android documentation!
 
 This pages list the complete API that this SDK is exposing to a client application.

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/common/CryptoTestHelper.kt

@@ -19,6 +19,7 @@ package org.matrix.android.sdk.common
 import android.os.SystemClock
 import android.util.Log
 import androidx.lifecycle.Observer
+import org.amshove.kluent.fail
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertNull
@@ -31,8 +32,16 @@ import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_MEGOLM
 import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_MEGOLM_BACKUP
 import org.matrix.android.sdk.api.extensions.orFalse
 import org.matrix.android.sdk.api.session.Session
+import org.matrix.android.sdk.api.session.crypto.MXCryptoError
+import org.matrix.android.sdk.api.session.crypto.crosssigning.KEYBACKUP_SECRET_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.crosssigning.MASTER_KEY_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.crosssigning.SELF_SIGNING_KEY_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.crosssigning.USER_SIGNING_KEY_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersion
 import org.matrix.android.sdk.api.session.crypto.keysbackup.MegolmBackupAuthData
 import org.matrix.android.sdk.api.session.crypto.keysbackup.MegolmBackupCreationInfo
+import org.matrix.android.sdk.api.session.crypto.keysbackup.extractCurveKeyFromRecoveryKey
+import org.matrix.android.sdk.api.session.crypto.model.OlmDecryptionResult
 import org.matrix.android.sdk.api.session.crypto.verification.IncomingSasVerificationTransaction
 import org.matrix.android.sdk.api.session.crypto.verification.OutgoingSasVerificationTransaction
 import org.matrix.android.sdk.api.session.crypto.verification.VerificationMethod
@@ -40,13 +49,19 @@ import org.matrix.android.sdk.api.session.crypto.verification.VerificationTxStat
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.toContent
+import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.getRoom
 import org.matrix.android.sdk.api.session.room.Room
 import org.matrix.android.sdk.api.session.room.model.Membership
 import org.matrix.android.sdk.api.session.room.model.RoomSummary
 import org.matrix.android.sdk.api.session.room.model.create.CreateRoomParams
+import org.matrix.android.sdk.api.session.room.model.message.MessageContent
 import org.matrix.android.sdk.api.session.room.roomSummaryQueryParams
+import org.matrix.android.sdk.api.session.securestorage.EmptyKeySigner
+import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
 import org.matrix.android.sdk.api.util.Optional
+import org.matrix.android.sdk.api.util.awaitCallback
+import org.matrix.android.sdk.api.util.toBase64NoPadding
 import java.util.UUID
 import kotlin.coroutines.Continuation
 import kotlin.coroutines.resume
@@ -188,17 +203,49 @@ class CryptoTestHelper(private val testHelper: CommonTestHelper) {
         val roomFromAlicePOV = aliceSession.getRoom(aliceRoomId)!!
 
         // Alice sends a message
-        testHelper.sendTextMessage(roomFromAlicePOV, messagesFromAlice[0], 1)
+        testHelper.sendTextMessage(roomFromAlicePOV, messagesFromAlice[0], 1).first().eventId.let { sentEventId ->
+            // ensure bob got it
+            ensureEventReceived(aliceRoomId, sentEventId, bobSession, true)
+        }
 
         // Bob send 3 messages
-        testHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[0], 1)
+        testHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[0], 1).first().eventId.let { sentEventId ->
-        testHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[1], 1)
+            // ensure alice got it
-        testHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[2], 1)
+            ensureEventReceived(aliceRoomId, sentEventId, aliceSession, true)
+        }
+
+        testHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[1], 1).first().eventId.let { sentEventId ->
+            // ensure alice got it
+            ensureEventReceived(aliceRoomId, sentEventId, aliceSession, true)
+        }
+        testHelper.sendTextMessage(roomFromBobPOV, messagesFromBob[2], 1).first().eventId.let { sentEventId ->
+            // ensure alice got it
+            ensureEventReceived(aliceRoomId, sentEventId, aliceSession, true)
+        }
+
         // Alice sends a message
-        testHelper.sendTextMessage(roomFromAlicePOV, messagesFromAlice[1], 1)
+        testHelper.sendTextMessage(roomFromAlicePOV, messagesFromAlice[1], 1).first().eventId.let { sentEventId ->
+            // ensure bob got it
+            ensureEventReceived(aliceRoomId, sentEventId, bobSession, true)
+        }
         return cryptoTestData
     }
 
+    private fun ensureEventReceived(roomId: String, eventId: String, session: Session, andCanDecrypt: Boolean) {
+        testHelper.waitWithLatch { latch ->
+            testHelper.retryPeriodicallyWithLatch(latch) {
+                val timeLineEvent = session.getRoom(roomId)?.timelineService()?.getTimelineEvent(eventId)
+                if (andCanDecrypt) {
+                    timeLineEvent != null &&
+                            timeLineEvent.isEncrypted() &&
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
+                } else {
+                    timeLineEvent != null
+                }
+            }
+        }
+    }
+
     fun checkEncryptedEvent(event: Event, roomId: String, clearMessage: String, senderSession: Session) {
         assertEquals(EventType.ENCRYPTED, event.type)
         assertNotNull(event.content)
@@ -296,17 +343,104 @@ class CryptoTestHelper(private val testHelper: CommonTestHelper) {
         }
     }
 
+    /**
+     * Initialize cross-signing, set up megolm backup and save all in 4S
+     */
+    fun bootstrapSecurity(session: Session) {
+        initializeCrossSigning(session)
+        val ssssService = session.sharedSecretStorageService()
+        testHelper.runBlockingTest {
+            val keyInfo = ssssService.generateKey(
+                    UUID.randomUUID().toString(),
+                    null,
+                    "ssss_key",
+                    EmptyKeySigner()
+            )
+            ssssService.setDefaultKey(keyInfo.keyId)
+
+            ssssService.storeSecret(
+                    MASTER_KEY_SSSS_NAME,
+                    session.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.master!!,
+                    listOf(SharedSecretStorageService.KeyRef(keyInfo.keyId, keyInfo.keySpec))
+            )
+
+            ssssService.storeSecret(
+                    SELF_SIGNING_KEY_SSSS_NAME,
+                    session.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.selfSigned!!,
+                    listOf(SharedSecretStorageService.KeyRef(keyInfo.keyId, keyInfo.keySpec))
+            )
+
+            ssssService.storeSecret(
+                    USER_SIGNING_KEY_SSSS_NAME,
+                    session.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.user!!,
+                    listOf(SharedSecretStorageService.KeyRef(keyInfo.keyId, keyInfo.keySpec))
+            )
+
+            // set up megolm backup
+            val creationInfo = awaitCallback<MegolmBackupCreationInfo> {
+                session.cryptoService().keysBackupService().prepareKeysBackupVersion(null, null, it)
+            }
+            val version = awaitCallback<KeysVersion> {
+                session.cryptoService().keysBackupService().createKeysBackupVersion(creationInfo, it)
+            }
+            // Save it for gossiping
+            session.cryptoService().keysBackupService().saveBackupRecoveryKey(creationInfo.recoveryKey, version = version.version)
+
+            extractCurveKeyFromRecoveryKey(creationInfo.recoveryKey)?.toBase64NoPadding()?.let { secret ->
+                ssssService.storeSecret(
+                        KEYBACKUP_SECRET_SSSS_NAME,
+                        secret,
+                        listOf(SharedSecretStorageService.KeyRef(keyInfo.keyId, keyInfo.keySpec))
+                )
+            }
+        }
+    }
+
     fun verifySASCrossSign(alice: Session, bob: Session, roomId: String) {
         assertTrue(alice.cryptoService().crossSigningService().canCrossSign())
         assertTrue(bob.cryptoService().crossSigningService().canCrossSign())
 
-        val requestID = UUID.randomUUID().toString()
         val aliceVerificationService = alice.cryptoService().verificationService()
         val bobVerificationService = bob.cryptoService().verificationService()
 
+        val localId = UUID.randomUUID().toString()
+        aliceVerificationService.requestKeyVerificationInDMs(
+                localId = localId,
+                methods = listOf(VerificationMethod.SAS, VerificationMethod.QR_CODE_SCAN, VerificationMethod.QR_CODE_SHOW),
+                otherUserId = bob.myUserId,
+                roomId = roomId
+        ).transactionId
+
+        testHelper.waitWithLatch {
+            testHelper.retryPeriodicallyWithLatch(it) {
+                bobVerificationService.getExistingVerificationRequests(alice.myUserId).firstOrNull {
+                    it.requestInfo?.fromDevice == alice.sessionParams.deviceId
+                } != null
+            }
+        }
+        val incomingRequest = bobVerificationService.getExistingVerificationRequests(alice.myUserId).first {
+            it.requestInfo?.fromDevice == alice.sessionParams.deviceId
+        }
+        bobVerificationService.readyPendingVerification(listOf(VerificationMethod.SAS), alice.myUserId, incomingRequest.transactionId!!)
+
+        var requestID: String? = null
+        // wait for it to be readied
+        testHelper.waitWithLatch {
+            testHelper.retryPeriodicallyWithLatch(it) {
+                val outgoingRequest = aliceVerificationService.getExistingVerificationRequests(bob.myUserId)
+                        .firstOrNull { it.localId == localId }
+                if (outgoingRequest?.isReady == true) {
+                    requestID = outgoingRequest.transactionId!!
+                    true
+                } else {
+                    false
+                }
+            }
+        }
+
         aliceVerificationService.beginKeyVerificationInDMs(
                 VerificationMethod.SAS,
-                requestID,
+                requestID!!,
                 roomId,
                 bob.myUserId,
                 bob.sessionParams.credentials.deviceId!!
@@ -316,23 +450,9 @@ class CryptoTestHelper(private val testHelper: CommonTestHelper) {
         var alicePovTx: OutgoingSasVerificationTransaction? = null
         var bobPovTx: IncomingSasVerificationTransaction? = null
 
-        // wait for alice to get the ready
         testHelper.waitWithLatch {
             testHelper.retryPeriodicallyWithLatch(it) {
-                bobPovTx = bobVerificationService.getExistingTransaction(alice.myUserId, requestID) as? IncomingSasVerificationTransaction
+                alicePovTx = aliceVerificationService.getExistingTransaction(bob.myUserId, requestID!!) as? OutgoingSasVerificationTransaction
-                Log.v("TEST", "== bobPovTx is ${alicePovTx?.uxState}")
-                if (bobPovTx?.state == VerificationTxState.OnStarted) {
-                    bobPovTx?.performAccept()
-                    true
-                } else {
-                    false
-                }
-            }
-        }
-
-        testHelper.waitWithLatch {
-            testHelper.retryPeriodicallyWithLatch(it) {
-                alicePovTx = aliceVerificationService.getExistingTransaction(bob.myUserId, requestID) as? OutgoingSasVerificationTransaction
                 Log.v("TEST", "== alicePovTx is ${alicePovTx?.uxState}")
                 alicePovTx?.state == VerificationTxState.ShortCodeReady
             }
@@ -340,7 +460,7 @@ class CryptoTestHelper(private val testHelper: CommonTestHelper) {
         // wait for alice to get the ready
         testHelper.waitWithLatch {
             testHelper.retryPeriodicallyWithLatch(it) {
-                bobPovTx = bobVerificationService.getExistingTransaction(alice.myUserId, requestID) as? IncomingSasVerificationTransaction
+                bobPovTx = bobVerificationService.getExistingTransaction(alice.myUserId, requestID!!) as? IncomingSasVerificationTransaction
                 Log.v("TEST", "== bobPovTx is ${alicePovTx?.uxState}")
                 if (bobPovTx?.state == VerificationTxState.OnStarted) {
                     bobPovTx?.performAccept()
@@ -392,4 +512,50 @@ class CryptoTestHelper(private val testHelper: CommonTestHelper) {
 
         return CryptoTestData(roomId, sessions)
     }
+
+    fun ensureCanDecrypt(sentEventIds: List<String>, session: Session, e2eRoomID: String, messagesText: List<String>) {
+        sentEventIds.forEachIndexed { index, sentEventId ->
+            testHelper.waitWithLatch { latch ->
+                testHelper.retryPeriodicallyWithLatch(latch) {
+                    val event = session.getRoom(e2eRoomID)!!.timelineService().getTimelineEvent(sentEventId)!!.root
+                    testHelper.runBlockingTest {
+                        try {
+                            session.cryptoService().decryptEvent(event, "").let { result ->
+                                event.mxDecryptionResult = OlmDecryptionResult(
+                                        payload = result.clearEvent,
+                                        senderKey = result.senderCurve25519Key,
+                                        keysClaimed = result.claimedEd25519Key?.let { mapOf("ed25519" to it) },
+                                        forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
+                                )
+                            }
+                        } catch (error: MXCryptoError) {
+                            // nop
+                        }
+                    }
+                    Log.v("TEST", "ensureCanDecrypt ${event.getClearType()} is ${event.getClearContent()}")
+                    event.getClearType() == EventType.MESSAGE &&
+                            messagesText[index] == event.getClearContent()?.toModel<MessageContent>()?.body
+                }
+            }
+        }
+    }
+
+    fun ensureCannotDecrypt(sentEventIds: List<String>, session: Session, e2eRoomID: String, expectedError: MXCryptoError.ErrorType? = null) {
+        sentEventIds.forEach { sentEventId ->
+            val event = session.getRoom(e2eRoomID)!!.timelineService().getTimelineEvent(sentEventId)!!.root
+            testHelper.runBlockingTest {
+                try {
+                    session.cryptoService().decryptEvent(event, "")
+                    fail("Should not be able to decrypt event")
+                } catch (error: MXCryptoError) {
+                    val errorType = (error as? MXCryptoError.Base)?.errorType
+                    if (expectedError == null) {
+                        assertNotNull(errorType)
+                    } else {
+                        assertEquals("Unexpected reason", expectedError, errorType)
+                    }
+                }
+            }
+        }
+    }
 }

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/E2eeSanityTests.kt

@@ -30,13 +30,21 @@ import org.junit.runners.MethodSorters
 import org.matrix.android.sdk.InstrumentedTest
 import org.matrix.android.sdk.api.session.Session
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
+import org.matrix.android.sdk.api.session.crypto.RequestResult
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersion
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersionResult
 import org.matrix.android.sdk.api.session.crypto.keysbackup.MegolmBackupCreationInfo
+import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.ImportRoomKeysResult
-import org.matrix.android.sdk.api.session.crypto.model.OlmDecryptionResult
+import org.matrix.android.sdk.api.session.crypto.verification.IncomingSasVerificationTransaction
+import org.matrix.android.sdk.api.session.crypto.verification.OutgoingSasVerificationTransaction
+import org.matrix.android.sdk.api.session.crypto.verification.PendingVerificationRequest
+import org.matrix.android.sdk.api.session.crypto.verification.VerificationMethod
+import org.matrix.android.sdk.api.session.crypto.verification.VerificationService
+import org.matrix.android.sdk.api.session.crypto.verification.VerificationTransaction
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
+import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.getRoom
 import org.matrix.android.sdk.api.session.getRoomSummary
@@ -52,15 +60,13 @@ import org.matrix.android.sdk.common.CryptoTestHelper
 import org.matrix.android.sdk.common.SessionTestParams
 import org.matrix.android.sdk.common.TestConstants
 import org.matrix.android.sdk.common.TestMatrixCallback
+import java.util.concurrent.CountDownLatch
 
 @RunWith(JUnit4::class)
 @FixMethodOrder(MethodSorters.JVM)
 @LargeTest
 class E2eeSanityTests : InstrumentedTest {
 
-    private val testHelper = CommonTestHelper(context())
-    private val cryptoTestHelper = CryptoTestHelper(testHelper)
-
     /**
      * Simple test that create an e2ee room.
      * Some new members are added, and a message is sent.
@@ -72,16 +78,24 @@ class E2eeSanityTests : InstrumentedTest {
      */
     @Test
     fun testSendingE2EEMessages() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
         val aliceSession = cryptoTestData.firstSession
         val e2eRoomID = cryptoTestData.roomId
 
         val aliceRoomPOV = aliceSession.getRoom(e2eRoomID)!!
+        // we want to disable key gossiping to just check initial sending of keys
+        aliceSession.cryptoService().enableKeyGossiping(false)
+        cryptoTestData.secondSession?.cryptoService()?.enableKeyGossiping(false)
 
         // add some more users and invite them
         val otherAccounts = listOf("benoit", "valere", "ganfra") // , "adam", "manu")
                 .map {
-                    testHelper.createAccount(it, SessionTestParams(true))
+                    testHelper.createAccount(it, SessionTestParams(true)).also {
+                        it.cryptoService().enableKeyGossiping(false)
+                    }
                 }
 
         Log.v("#E2E TEST", "All accounts created")
@@ -95,18 +109,18 @@ class E2eeSanityTests : InstrumentedTest {
 
         // All user should accept invite
         otherAccounts.forEach { otherSession ->
-            waitForAndAcceptInviteInRoom(otherSession, e2eRoomID)
+            waitForAndAcceptInviteInRoom(testHelper, otherSession, e2eRoomID)
             Log.v("#E2E TEST", "${otherSession.myUserId} joined room $e2eRoomID")
         }
 
         // check that alice see them as joined (not really necessary?)
-        ensureMembersHaveJoined(aliceSession, otherAccounts, e2eRoomID)
+        ensureMembersHaveJoined(testHelper, aliceSession, otherAccounts, e2eRoomID)
 
         Log.v("#E2E TEST", "All users have joined the room")
         Log.v("#E2E TEST", "Alice is sending the message")
 
         val text = "This is my message"
-        val sentEventId: String? = sendMessageInRoom(aliceRoomPOV, text)
+        val sentEventId: String? = sendMessageInRoom(testHelper, aliceRoomPOV, text)
         //        val sentEvent = testHelper.sendTextMessage(aliceRoomPOV, "Hello all", 1).first()
         Assert.assertTrue("Message should be sent", sentEventId != null)
 
@@ -114,10 +128,10 @@ class E2eeSanityTests : InstrumentedTest {
         otherAccounts.forEach { otherSession ->
             testHelper.waitWithLatch { latch ->
                 testHelper.retryPeriodicallyWithLatch(latch) {
-                    val timelineEvent = otherSession.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId!!)
+                    val timeLineEvent = otherSession.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId!!)
-                    timelineEvent != null &&
+                    timeLineEvent != null &&
-                            timelineEvent.isEncrypted() &&
+                            timeLineEvent.isEncrypted() &&
-                            timelineEvent.root.getClearType() == EventType.MESSAGE
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
                 }
             }
         }
@@ -136,10 +150,10 @@ class E2eeSanityTests : InstrumentedTest {
         }
 
         newAccount.forEach {
-            waitForAndAcceptInviteInRoom(it, e2eRoomID)
+            waitForAndAcceptInviteInRoom(testHelper, it, e2eRoomID)
         }
 
-        ensureMembersHaveJoined(aliceSession, newAccount, e2eRoomID)
+        ensureMembersHaveJoined(testHelper, aliceSession, newAccount, e2eRoomID)
 
         // wait a bit
         testHelper.runBlockingTest {
@@ -164,7 +178,7 @@ class E2eeSanityTests : InstrumentedTest {
         Log.v("#E2E TEST", "Alice sends a new message")
 
         val secondMessage = "2 This is my message"
-        val secondSentEventId: String? = sendMessageInRoom(aliceRoomPOV, secondMessage)
+        val secondSentEventId: String? = sendMessageInRoom(testHelper, aliceRoomPOV, secondMessage)
 
         // new members should be able to decrypt it
         newAccount.forEach { otherSession ->
@@ -188,6 +202,14 @@ class E2eeSanityTests : InstrumentedTest {
         cryptoTestData.cleanUp(testHelper)
     }
 
+    @Test
+    fun testKeyGossipingIsEnabledByDefault() {
+        val testHelper = CommonTestHelper(context())
+        val session = testHelper.createAccount("alice", SessionTestParams(true))
+        Assert.assertTrue("Key gossiping should be enabled by default", session.cryptoService().isKeyGossipingEnabled())
+        testHelper.signOutAndClose(session)
+    }
+
     /**
      * Quick test for basic key backup
      * 1. Create e2e between Alice and Bob
@@ -204,6 +226,9 @@ class E2eeSanityTests : InstrumentedTest {
      */
     @Test
     fun testBasicBackupImport() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
         val aliceSession = cryptoTestData.firstSession
         val bobSession = cryptoTestData.secondSession!!
@@ -227,16 +252,16 @@ class E2eeSanityTests : InstrumentedTest {
         val sentEventIds = mutableListOf<String>()
         val messagesText = listOf("1. Hello", "2. Bob", "3. Good morning")
         messagesText.forEach { text ->
-            val sentEventId = sendMessageInRoom(aliceRoomPOV, text)!!.also {
+            val sentEventId = sendMessageInRoom(testHelper, aliceRoomPOV, text)!!.also {
                 sentEventIds.add(it)
             }
 
             testHelper.waitWithLatch { latch ->
                 testHelper.retryPeriodicallyWithLatch(latch) {
-                    val timelineEvent = bobSession.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId)
+                    val timeLineEvent = bobSession.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId)
-                    timelineEvent != null &&
+                    timeLineEvent != null &&
-                            timelineEvent.isEncrypted() &&
+                            timeLineEvent.isEncrypted() &&
-                            timelineEvent.root.getClearType() == EventType.MESSAGE
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
                 }
             }
             // we want more so let's discard the session
@@ -289,22 +314,23 @@ class E2eeSanityTests : InstrumentedTest {
             }
         }
         // after initial sync events are not decrypted, so we have to try manually
-        ensureCannotDecrypt(sentEventIds, newBobSession, e2eRoomID, MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID)
+        cryptoTestHelper.ensureCannotDecrypt(sentEventIds, newBobSession, e2eRoomID, MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID)
 
         // Let's now import keys from backup
 
-        newBobSession.cryptoService().keysBackupService().let { keysBackupService ->
+        newBobSession.cryptoService().keysBackupService().let { kbs ->
             val keyVersionResult = testHelper.doSync<KeysVersionResult?> {
-                keysBackupService.getVersion(version.version, it)
+                kbs.getVersion(version.version, it)
             }
 
             val importedResult = testHelper.doSync<ImportRoomKeysResult> {
-                keysBackupService.restoreKeyBackupWithPassword(
+                kbs.restoreKeyBackupWithPassword(
                         keyVersionResult!!,
                         keyBackupPassword,
                         null,
                         null,
-                        null, it
+                        null,
+                        it
                 )
             }
 
@@ -312,7 +338,7 @@ class E2eeSanityTests : InstrumentedTest {
         }
 
         // ensure bob can now decrypt
-        ensureCanDecrypt(sentEventIds, newBobSession, e2eRoomID, messagesText)
+        cryptoTestHelper.ensureCanDecrypt(sentEventIds, newBobSession, e2eRoomID, messagesText)
 
         testHelper.signOutAndClose(newBobSession)
     }
@@ -323,6 +349,9 @@ class E2eeSanityTests : InstrumentedTest {
      */
     @Test
     fun testSimpleGossip() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
         val aliceSession = cryptoTestData.firstSession
         val bobSession = cryptoTestData.secondSession!!
@@ -330,30 +359,28 @@ class E2eeSanityTests : InstrumentedTest {
 
         val aliceRoomPOV = aliceSession.getRoom(e2eRoomID)!!
 
-        cryptoTestHelper.initializeCrossSigning(bobSession)
-
         // let's send a few message to bob
         val sentEventIds = mutableListOf<String>()
         val messagesText = listOf("1. Hello", "2. Bob")
 
         Log.v("#E2E TEST", "Alice sends some messages")
         messagesText.forEach { text ->
-            val sentEventId = sendMessageInRoom(aliceRoomPOV, text)!!.also {
+            val sentEventId = sendMessageInRoom(testHelper, aliceRoomPOV, text)!!.also {
                 sentEventIds.add(it)
             }
 
             testHelper.waitWithLatch { latch ->
                 testHelper.retryPeriodicallyWithLatch(latch) {
-                    val timelineEvent = bobSession.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId)
+                    val timeLineEvent = bobSession.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId)
-                    timelineEvent != null &&
+                    timeLineEvent != null &&
-                            timelineEvent.isEncrypted() &&
+                            timeLineEvent.isEncrypted() &&
-                            timelineEvent.root.getClearType() == EventType.MESSAGE
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
                 }
             }
         }
 
         // Ensure bob can decrypt
-        ensureIsDecrypted(sentEventIds, bobSession, e2eRoomID)
+        ensureIsDecrypted(testHelper, sentEventIds, bobSession, e2eRoomID)
 
         // Let's now add a new bob session
         // Create a new session for bob
@@ -363,7 +390,11 @@ class E2eeSanityTests : InstrumentedTest {
         // check that new bob can't currently decrypt
         Log.v("#E2E TEST", "check that new bob can't currently decrypt")
 
-        ensureCannotDecrypt(sentEventIds, newBobSession, e2eRoomID, MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID)
+        cryptoTestHelper.ensureCannotDecrypt(sentEventIds, newBobSession, e2eRoomID, null)
+//        newBobSession.cryptoService().getOutgoingRoomKeyRequests()
+//                .firstOrNull {
+//                    it.sessionId ==
+//                }
 
         // Try to request
         sentEventIds.forEach { sentEventId ->
@@ -372,12 +403,34 @@ class E2eeSanityTests : InstrumentedTest {
         }
 
         // wait a bit
-        testHelper.runBlockingTest {
+        // we need to wait a couple of syncs to let sharing occurs
-            delay(10_000)
+//        testHelper.waitFewSyncs(newBobSession, 6)
-        }
 
         // Ensure that new bob still can't decrypt (keys must have been withheld)
-        ensureCannotDecrypt(sentEventIds, newBobSession, e2eRoomID, MXCryptoError.ErrorType.KEYS_WITHHELD)
+        sentEventIds.forEach { sentEventId ->
+            val megolmSessionId = newBobSession.getRoom(e2eRoomID)!!
+                    .getTimelineEvent(sentEventId)!!
+                    .root.content.toModel<EncryptedEventContent>()!!.sessionId
+            testHelper.waitWithLatch { latch ->
+                testHelper.retryPeriodicallyWithLatch(latch) {
+                    val aliceReply = newBobSession.cryptoService().getOutgoingRoomKeyRequests()
+                            .first {
+                                it.sessionId == megolmSessionId &&
+                                        it.roomId == e2eRoomID
+                            }
+                            .results.also {
+                                Log.w("##TEST", "result list is $it")
+                            }
+                            .firstOrNull { it.userId == aliceSession.myUserId }
+                            ?.result
+                    aliceReply != null &&
+                            aliceReply is RequestResult.Failure &&
+                            WithHeldCode.UNAUTHORISED == aliceReply.code
+                }
+            }
+        }
+
+        cryptoTestHelper.ensureCannotDecrypt(sentEventIds, newBobSession, e2eRoomID, null)
 
         // Now mark new bob session as verified
 
@@ -390,12 +443,7 @@ class E2eeSanityTests : InstrumentedTest {
             newBobSession.cryptoService().reRequestRoomKeyForEvent(event)
         }
 
-        // wait a bit
+        cryptoTestHelper.ensureCanDecrypt(sentEventIds, newBobSession, e2eRoomID, messagesText)
-        testHelper.runBlockingTest {
-            delay(10_000)
-        }
-
-        ensureCanDecrypt(sentEventIds, newBobSession, e2eRoomID, messagesText)
 
         cryptoTestData.cleanUp(testHelper)
         testHelper.signOutAndClose(newBobSession)
@@ -406,6 +454,9 @@ class E2eeSanityTests : InstrumentedTest {
      */
     @Test
     fun testForwardBetterKey() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
         val aliceSession = cryptoTestData.firstSession
         val bobSessionWithBetterKey = cryptoTestData.secondSession!!
@@ -413,35 +464,33 @@ class E2eeSanityTests : InstrumentedTest {
 
         val aliceRoomPOV = aliceSession.getRoom(e2eRoomID)!!
 
-        cryptoTestHelper.initializeCrossSigning(bobSessionWithBetterKey)
-
         // let's send a few message to bob
         var firstEventId: String
         val firstMessage = "1. Hello"
 
         Log.v("#E2E TEST", "Alice sends some messages")
         firstMessage.let { text ->
-            firstEventId = sendMessageInRoom(aliceRoomPOV, text)!!
+            firstEventId = sendMessageInRoom(testHelper, aliceRoomPOV, text)!!
 
             testHelper.waitWithLatch { latch ->
                 testHelper.retryPeriodicallyWithLatch(latch) {
-                    val timelineEvent = bobSessionWithBetterKey.getRoom(e2eRoomID)?.getTimelineEvent(firstEventId)
+                    val timeLineEvent = bobSessionWithBetterKey.getRoom(e2eRoomID)?.getTimelineEvent(firstEventId)
-                    timelineEvent != null &&
+                    timeLineEvent != null &&
-                            timelineEvent.isEncrypted() &&
+                            timeLineEvent.isEncrypted() &&
-                            timelineEvent.root.getClearType() == EventType.MESSAGE
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
                 }
             }
         }
 
         // Ensure bob can decrypt
-        ensureIsDecrypted(listOf(firstEventId), bobSessionWithBetterKey, e2eRoomID)
+        ensureIsDecrypted(testHelper, listOf(firstEventId), bobSessionWithBetterKey, e2eRoomID)
 
         // Let's add a new unverified session from bob
         val newBobSession = testHelper.logIntoAccount(bobSessionWithBetterKey.myUserId, SessionTestParams(true))
 
         // check that new bob can't currently decrypt
         Log.v("#E2E TEST", "check that new bob can't currently decrypt")
-        ensureCannotDecrypt(listOf(firstEventId), newBobSession, e2eRoomID, MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID)
+        cryptoTestHelper.ensureCannotDecrypt(listOf(firstEventId), newBobSession, e2eRoomID, null)
 
         // Now let alice send a new message. this time the new bob session will be able to decrypt
         var secondEventId: String
@@ -449,14 +498,14 @@ class E2eeSanityTests : InstrumentedTest {
 
         Log.v("#E2E TEST", "Alice sends some messages")
         secondMessage.let { text ->
-            secondEventId = sendMessageInRoom(aliceRoomPOV, text)!!
+            secondEventId = sendMessageInRoom(testHelper, aliceRoomPOV, text)!!
 
             testHelper.waitWithLatch { latch ->
                 testHelper.retryPeriodicallyWithLatch(latch) {
-                    val timelineEvent = newBobSession.getRoom(e2eRoomID)?.getTimelineEvent(secondEventId)
+                    val timeLineEvent = newBobSession.getRoom(e2eRoomID)?.getTimelineEvent(secondEventId)
-                    timelineEvent != null &&
+                    timeLineEvent != null &&
-                            timelineEvent.isEncrypted() &&
+                            timeLineEvent.isEncrypted() &&
-                            timelineEvent.root.getClearType() == EventType.MESSAGE
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
                 }
             }
         }
@@ -475,9 +524,7 @@ class E2eeSanityTests : InstrumentedTest {
             try {
                 newBobSession.cryptoService().decryptEvent(firstEventNewBobPov.root, "")
                 fail("Should not be able to decrypt event")
-            } catch (error: MXCryptoError) {
+            } catch (_: MXCryptoError) {
-                val errorType = (error as? MXCryptoError.Base)?.errorType
-                assertEquals(MXCryptoError.ErrorType.UNKNOWN_MESSAGE_INDEX, errorType)
             }
         }
 
@@ -499,41 +546,45 @@ class E2eeSanityTests : InstrumentedTest {
                 .markedLocallyAsManuallyVerified(bobSessionWithBetterKey.myUserId, bobSessionWithBetterKey.sessionParams.deviceId!!)
 
         // now let new session request
-        newBobSession.cryptoService().requestRoomKeyForEvent(firstEventNewBobPov.root)
+        newBobSession.cryptoService().reRequestRoomKeyForEvent(firstEventNewBobPov.root)
 
-        // wait a bit
+        // We need to wait for the key request to be sent out and then a reply to be received
-        testHelper.runBlockingTest {
-            delay(10_000)
-        }
 
         // old session should have shared the key at earliest known index now
         // we should be able to decrypt both
-        testHelper.runBlockingTest {
+        testHelper.waitWithLatch {
-            try {
+            testHelper.retryPeriodicallyWithLatch(it) {
-                newBobSession.cryptoService().decryptEvent(firstEventNewBobPov.root, "")
+                val canDecryptFirst = try {
-            } catch (error: MXCryptoError) {
+                    testHelper.runBlockingTest {
-                fail("Should be able to decrypt first event now $error")
+                        newBobSession.cryptoService().decryptEvent(firstEventNewBobPov.root, "")
-            }
+                    }
-        }
+                    true
-        testHelper.runBlockingTest {
+                } catch (error: MXCryptoError) {
-            try {
+                    false
-                newBobSession.cryptoService().decryptEvent(secondEventNewBobPov.root, "")
+                }
-            } catch (error: MXCryptoError) {
+                val canDecryptSecond = try {
-                fail("Should be able to decrypt event $error")
+                    testHelper.runBlockingTest {
+                        newBobSession.cryptoService().decryptEvent(secondEventNewBobPov.root, "")
+                    }
+                    true
+                } catch (error: MXCryptoError) {
+                    false
+                }
+                canDecryptFirst && canDecryptSecond
             }
         }
 
-        cryptoTestData.cleanUp(testHelper)
+        testHelper.signOutAndClose(aliceSession)
+        testHelper.signOutAndClose(bobSessionWithBetterKey)
         testHelper.signOutAndClose(newBobSession)
     }
 
-    private fun sendMessageInRoom(aliceRoomPOV: Room, text: String): String? {
+    private fun sendMessageInRoom(testHelper: CommonTestHelper, aliceRoomPOV: Room, text: String): String? {
         aliceRoomPOV.sendService().sendTextMessage(text)
         var sentEventId: String? = null
         testHelper.waitWithLatch(4 * TestConstants.timeOutMillis) { latch ->
             val timeline = aliceRoomPOV.timelineService().createTimeline(null, TimelineSettings(60))
             timeline.start()
-
             testHelper.retryPeriodicallyWithLatch(latch) {
                 val decryptedMsg = timeline.getSnapshot()
                         .filter { it.root.getClearType() == EventType.MESSAGE }
@@ -552,7 +603,157 @@ class E2eeSanityTests : InstrumentedTest {
         return sentEventId
     }
 
-    private fun ensureMembersHaveJoined(aliceSession: Session, otherAccounts: List<Session>, e2eRoomID: String) {
+    /**
+     * Test that if a better key is forwared (lower index, it is then used)
+     */
+    @Test
+    fun testSelfInteractiveVerificationAndGossip() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
+        val aliceSession = testHelper.createAccount("alice", SessionTestParams(true))
+        cryptoTestHelper.bootstrapSecurity(aliceSession)
+
+        // now let's create a new login from alice
+
+        val aliceNewSession = testHelper.logIntoAccount(aliceSession.myUserId, SessionTestParams(true))
+
+        val oldCompleteLatch = CountDownLatch(1)
+        lateinit var oldCode: String
+        aliceSession.cryptoService().verificationService().addListener(object : VerificationService.Listener {
+
+            override fun verificationRequestUpdated(pr: PendingVerificationRequest) {
+                val readyInfo = pr.readyInfo
+                if (readyInfo != null) {
+                    aliceSession.cryptoService().verificationService().beginKeyVerification(
+                            VerificationMethod.SAS,
+                            aliceSession.myUserId,
+                            readyInfo.fromDevice,
+                            readyInfo.transactionId
+
+                    )
+                }
+            }
+
+            override fun transactionUpdated(tx: VerificationTransaction) {
+                Log.d("##TEST", "exitsingPov: $tx")
+                val sasTx = tx as OutgoingSasVerificationTransaction
+                when (sasTx.uxState) {
+                    OutgoingSasVerificationTransaction.UxState.SHOW_SAS -> {
+                        // for the test we just accept?
+                        oldCode = sasTx.getDecimalCodeRepresentation()
+                        sasTx.userHasVerifiedShortCode()
+                    }
+                    OutgoingSasVerificationTransaction.UxState.VERIFIED -> {
+                        // we can release this latch?
+                        oldCompleteLatch.countDown()
+                    }
+                    else                                                -> Unit
+                }
+            }
+        })
+
+        val newCompleteLatch = CountDownLatch(1)
+        lateinit var newCode: String
+        aliceNewSession.cryptoService().verificationService().addListener(object : VerificationService.Listener {
+
+            override fun verificationRequestCreated(pr: PendingVerificationRequest) {
+                // let's ready
+                aliceNewSession.cryptoService().verificationService().readyPendingVerification(
+                        listOf(VerificationMethod.SAS, VerificationMethod.QR_CODE_SCAN, VerificationMethod.QR_CODE_SHOW),
+                        aliceSession.myUserId,
+                        pr.transactionId!!
+                )
+            }
+
+            var matchOnce = true
+            override fun transactionUpdated(tx: VerificationTransaction) {
+                Log.d("##TEST", "newPov: $tx")
+
+                val sasTx = tx as IncomingSasVerificationTransaction
+                when (sasTx.uxState) {
+                    IncomingSasVerificationTransaction.UxState.SHOW_ACCEPT -> {
+                        // no need to accept as there was a request first it will auto accept
+                    }
+                    IncomingSasVerificationTransaction.UxState.SHOW_SAS    -> {
+                        if (matchOnce) {
+                            sasTx.userHasVerifiedShortCode()
+                            newCode = sasTx.getDecimalCodeRepresentation()
+                            matchOnce = false
+                        }
+                    }
+                    IncomingSasVerificationTransaction.UxState.VERIFIED    -> {
+                        newCompleteLatch.countDown()
+                    }
+                    else                                                   -> Unit
+                }
+            }
+        })
+
+        // initiate self verification
+        aliceSession.cryptoService().verificationService().requestKeyVerification(
+                listOf(VerificationMethod.SAS, VerificationMethod.QR_CODE_SCAN, VerificationMethod.QR_CODE_SHOW),
+                aliceNewSession.myUserId,
+                listOf(aliceNewSession.sessionParams.deviceId!!)
+        )
+        testHelper.await(oldCompleteLatch)
+        testHelper.await(newCompleteLatch)
+        assertEquals("Decimal code should have matched", oldCode, newCode)
+
+        // Assert that devices are verified
+        val newDeviceFromOldPov: CryptoDeviceInfo? = aliceSession.cryptoService().getDeviceInfo(aliceSession.myUserId, aliceNewSession.sessionParams.deviceId)
+        val oldDeviceFromNewPov: CryptoDeviceInfo? = aliceSession.cryptoService().getDeviceInfo(aliceSession.myUserId, aliceSession.sessionParams.deviceId)
+
+        Assert.assertTrue("new device should be verified from old point of view", newDeviceFromOldPov!!.isVerified)
+        Assert.assertTrue("old device should be verified from new point of view", oldDeviceFromNewPov!!.isVerified)
+
+        // wait for secret gossiping to happen
+        testHelper.waitWithLatch { latch ->
+            testHelper.retryPeriodicallyWithLatch(latch) {
+                aliceNewSession.cryptoService().crossSigningService().allPrivateKeysKnown()
+            }
+        }
+
+        testHelper.waitWithLatch { latch ->
+            testHelper.retryPeriodicallyWithLatch(latch) {
+                aliceNewSession.cryptoService().keysBackupService().getKeyBackupRecoveryKeyInfo() != null
+            }
+        }
+
+        assertEquals(
+                "MSK Private parts should be the same",
+                aliceSession.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.master,
+                aliceNewSession.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.master
+        )
+        assertEquals(
+                "USK Private parts should be the same",
+                aliceSession.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.user,
+                aliceNewSession.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.user
+        )
+
+        assertEquals(
+                "SSK Private parts should be the same",
+                aliceSession.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.selfSigned,
+                aliceNewSession.cryptoService().crossSigningService().getCrossSigningPrivateKeys()!!.selfSigned
+        )
+
+        // Let's check that we have the megolm backup key
+        assertEquals(
+                "Megolm key should be the same",
+                aliceSession.cryptoService().keysBackupService().getKeyBackupRecoveryKeyInfo()!!.recoveryKey,
+                aliceNewSession.cryptoService().keysBackupService().getKeyBackupRecoveryKeyInfo()!!.recoveryKey
+        )
+        assertEquals(
+                "Megolm version should be the same",
+                aliceSession.cryptoService().keysBackupService().getKeyBackupRecoveryKeyInfo()!!.version,
+                aliceNewSession.cryptoService().keysBackupService().getKeyBackupRecoveryKeyInfo()!!.version
+        )
+
+        testHelper.signOutAndClose(aliceSession)
+        testHelper.signOutAndClose(aliceNewSession)
+    }
+
+    private fun ensureMembersHaveJoined(testHelper: CommonTestHelper, aliceSession: Session, otherAccounts: List<Session>, e2eRoomID: String) {
         testHelper.waitWithLatch { latch ->
             testHelper.retryPeriodicallyWithLatch(latch) {
                 otherAccounts.map {
@@ -564,7 +765,7 @@ class E2eeSanityTests : InstrumentedTest {
         }
     }
 
-    private fun waitForAndAcceptInviteInRoom(otherSession: Session, e2eRoomID: String) {
+    private fun waitForAndAcceptInviteInRoom(testHelper: CommonTestHelper, otherSession: Session, e2eRoomID: String) {
         testHelper.waitWithLatch { latch ->
             testHelper.retryPeriodicallyWithLatch(latch) {
                 val roomSummary = otherSession.getRoomSummary(e2eRoomID)
@@ -576,7 +777,8 @@ class E2eeSanityTests : InstrumentedTest {
             }
         }
 
-        testHelper.runBlockingTest(60_000) {
+        // not sure why it's taking so long :/
+        testHelper.runBlockingTest(90_000) {
             Log.v("#E2E TEST", "${otherSession.myUserId} tries to join room $e2eRoomID")
             try {
                 otherSession.roomService().joinRoom(e2eRoomID)
@@ -594,59 +796,14 @@ class E2eeSanityTests : InstrumentedTest {
         }
     }
 
-    private fun ensureCanDecrypt(sentEventIds: MutableList<String>, session: Session, e2eRoomID: String, messagesText: List<String>) {
+    private fun ensureIsDecrypted(testHelper: CommonTestHelper, sentEventIds: List<String>, session: Session, e2eRoomID: String) {
-        sentEventIds.forEachIndexed { index, sentEventId ->
-            testHelper.waitWithLatch { latch ->
-                testHelper.retryPeriodicallyWithLatch(latch) {
-                    val event = session.getRoom(e2eRoomID)!!.getTimelineEvent(sentEventId)!!.root
-                    testHelper.runBlockingTest {
-                        try {
-                            session.cryptoService().decryptEvent(event, "").let { result ->
-                                event.mxDecryptionResult = OlmDecryptionResult(
-                                        payload = result.clearEvent,
-                                        senderKey = result.senderCurve25519Key,
-                                        keysClaimed = result.claimedEd25519Key?.let { mapOf("ed25519" to it) },
-                                        forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain
-                                )
-                            }
-                        } catch (error: MXCryptoError) {
-                            // nop
-                        }
-                    }
-                    event.getClearType() == EventType.MESSAGE &&
-                            messagesText[index] == event.getClearContent()?.toModel<MessageContent>()?.body
-                }
-            }
-        }
-    }
-
-    private fun ensureIsDecrypted(sentEventIds: List<String>, session: Session, e2eRoomID: String) {
         testHelper.waitWithLatch { latch ->
             sentEventIds.forEach { sentEventId ->
                 testHelper.retryPeriodicallyWithLatch(latch) {
-                    val timelineEvent = session.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId)
+                    val timeLineEvent = session.getRoom(e2eRoomID)?.getTimelineEvent(sentEventId)
-                    timelineEvent != null &&
+                    timeLineEvent != null &&
-                            timelineEvent.isEncrypted() &&
+                            timeLineEvent.isEncrypted() &&
-                            timelineEvent.root.getClearType() == EventType.MESSAGE
+                            timeLineEvent.root.getClearType() == EventType.MESSAGE
-                }
-            }
-        }
-    }
-
-    private fun ensureCannotDecrypt(sentEventIds: List<String>, newBobSession: Session, e2eRoomID: String, expectedError: MXCryptoError.ErrorType?) {
-        sentEventIds.forEach { sentEventId ->
-            val event = newBobSession.getRoom(e2eRoomID)!!.getTimelineEvent(sentEventId)!!.root
-            testHelper.runBlockingTest {
-                try {
-                    newBobSession.cryptoService().decryptEvent(event, "")
-                    fail("Should not be able to decrypt event")
-                } catch (error: MXCryptoError) {
-                    val errorType = (error as? MXCryptoError.Base)?.errorType
-                    if (expectedError == null) {
-                        Assert.assertNotNull(errorType)
-                    } else {
-                        assertEquals(expectedError, errorType, "Message expected to be UISI")
-                    }
                 }
             }
         }

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/PreShareKeysTest.kt

@@ -27,7 +27,6 @@ import org.junit.runners.MethodSorters
 import org.matrix.android.sdk.InstrumentedTest
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
-import org.matrix.android.sdk.api.session.events.model.content.RoomKeyContent
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.getRoom
 import org.matrix.android.sdk.api.session.room.getTimelineEvent
@@ -51,10 +50,7 @@ class PreShareKeysTest : InstrumentedTest {
         // clear any outbound session
         aliceSession.cryptoService().discardOutboundSession(e2eRoomID)
 
-        val preShareCount = bobSession.cryptoService().getGossipingEvents().count {
+        val preShareCount = bobSession.cryptoService().keysBackupService().getTotalNumbersOfKeys()
-            it.senderId == aliceSession.myUserId &&
-                    it.getClearType() == EventType.ROOM_KEY
-        }
 
         assertEquals("Bob should not have receive any key from alice at this point", 0, preShareCount)
         Log.d("#Test", "Room Key Received from alice $preShareCount")
@@ -66,23 +62,23 @@ class PreShareKeysTest : InstrumentedTest {
 
         testHelper.waitWithLatch { latch ->
             testHelper.retryPeriodicallyWithLatch(latch) {
-                val newGossipCount = bobSession.cryptoService().getGossipingEvents().count {
+                val newKeysCount = bobSession.cryptoService().keysBackupService().getTotalNumbersOfKeys()
-                    it.senderId == aliceSession.myUserId &&
+                newKeysCount > preShareCount
-                            it.getClearType() == EventType.ROOM_KEY
-                }
-                newGossipCount > preShareCount
             }
         }
 
-        val latest = bobSession.cryptoService().getGossipingEvents().lastOrNull {
+        val aliceCryptoStore = (aliceSession.cryptoService() as DefaultCryptoService).cryptoStoreForTesting
-            it.senderId == aliceSession.myUserId &&
+        val aliceOutboundSessionInRoom = aliceCryptoStore.getCurrentOutboundGroupSessionForRoom(e2eRoomID)!!.outboundGroupSession.sessionIdentifier()
-                    it.getClearType() == EventType.ROOM_KEY
-        }
 
-        val content = latest?.getClearContent().toModel<RoomKeyContent>()
+        val bobCryptoStore = (bobSession.cryptoService() as DefaultCryptoService).cryptoStoreForTesting
-        assertNotNull("Bob should have received and decrypted a room key event from alice", content)
+        val aliceDeviceBobPov = bobCryptoStore.getUserDevice(aliceSession.myUserId, aliceSession.sessionParams.deviceId!!)!!
-        assertEquals("Wrong room", e2eRoomID, content!!.roomId)
+        val bobInboundForAlice = bobCryptoStore.getInboundGroupSession(aliceOutboundSessionInRoom, aliceDeviceBobPov.identityKey()!!)
-        val megolmSessionId = content.sessionId!!
+        assertNotNull("Bob should have received and decrypted a room key event from alice", bobInboundForAlice)
+        assertEquals("Wrong room", e2eRoomID, bobInboundForAlice!!.roomId)
+
+        val megolmSessionId = bobInboundForAlice.olmInboundGroupSession!!.sessionIdentifier()
+
+        assertEquals("Wrong session", aliceOutboundSessionInRoom, megolmSessionId)
 
         val sharedIndex = aliceSession.cryptoService().getSharedWithInfo(e2eRoomID, megolmSessionId)
                 .getObject(bobSession.myUserId, bobSession.sessionParams.deviceId)

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/gossiping/KeyShareTests.kt

@@ -19,59 +19,45 @@ package org.matrix.android.sdk.internal.crypto.gossiping
 import android.util.Log
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
-import junit.framework.TestCase.assertEquals
 import junit.framework.TestCase.assertNotNull
 import junit.framework.TestCase.assertTrue
 import junit.framework.TestCase.fail
+import org.amshove.kluent.internal.assertEquals
 import org.junit.Assert
+import org.junit.Assert.assertNull
 import org.junit.FixMethodOrder
-import org.junit.Ignore
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.MethodSorters
 import org.matrix.android.sdk.InstrumentedTest
-import org.matrix.android.sdk.api.auth.UIABaseAuth
+import org.matrix.android.sdk.api.session.crypto.OutgoingRoomKeyRequestState
-import org.matrix.android.sdk.api.auth.UserInteractiveAuthInterceptor
+import org.matrix.android.sdk.api.session.crypto.RequestResult
-import org.matrix.android.sdk.api.auth.UserPasswordAuth
-import org.matrix.android.sdk.api.auth.registration.RegistrationFlowResponse
-import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.crypto.crosssigning.DeviceTrustLevel
-import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersion
-import org.matrix.android.sdk.api.session.crypto.keysbackup.MegolmBackupCreationInfo
-import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
-import org.matrix.android.sdk.api.session.crypto.model.GossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.verification.IncomingSasVerificationTransaction
-import org.matrix.android.sdk.api.session.crypto.verification.SasVerificationTransaction
-import org.matrix.android.sdk.api.session.crypto.verification.VerificationMethod
-import org.matrix.android.sdk.api.session.crypto.verification.VerificationService
-import org.matrix.android.sdk.api.session.crypto.verification.VerificationTransaction
-import org.matrix.android.sdk.api.session.crypto.verification.VerificationTxState
 import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
+import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.getRoom
 import org.matrix.android.sdk.api.session.room.getTimelineEvent
 import org.matrix.android.sdk.api.session.room.model.RoomDirectoryVisibility
 import org.matrix.android.sdk.api.session.room.model.create.CreateRoomParams
-import org.matrix.android.sdk.api.session.room.model.message.MessageContent
+import org.matrix.android.sdk.api.session.room.timeline.getLastMessageContent
 import org.matrix.android.sdk.common.CommonTestHelper
+import org.matrix.android.sdk.common.CryptoTestHelper
 import org.matrix.android.sdk.common.SessionTestParams
 import org.matrix.android.sdk.common.TestConstants
-import kotlin.coroutines.Continuation
-import kotlin.coroutines.resume
 
 @RunWith(AndroidJUnit4::class)
 @FixMethodOrder(MethodSorters.JVM)
 @LargeTest
 class KeyShareTests : InstrumentedTest {
 
-    private val commonTestHelper = CommonTestHelper(context())
-
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun test_DoNotSelfShareIfNotTrusted() {
+        val commonTestHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(commonTestHelper)
+
         val aliceSession = commonTestHelper.createAccount(TestConstants.USER_ALICE, SessionTestParams(true))
+        Log.v("TEST", "=======> AliceSession 1 is ${aliceSession.sessionParams.deviceId}")
 
         // Create an encrypted room and add a message
         val roomId = commonTestHelper.runBlockingTest {
@@ -86,11 +72,18 @@ class KeyShareTests : InstrumentedTest {
         assertNotNull(room)
         Thread.sleep(4_000)
         assertTrue(room?.roomCryptoService()?.isEncrypted() == true)
-        val sentEventId = commonTestHelper.sendTextMessage(room!!, "My Message", 1).first().eventId
 
-        // Open a new sessionx
+        val sentEvent = commonTestHelper.sendTextMessage(room!!, "My Message", 1).first()
+        val sentEventId = sentEvent.eventId
+        val sentEventText = sentEvent.getLastMessageContent()?.body
 
-        val aliceSession2 = commonTestHelper.logIntoAccount(aliceSession.myUserId, SessionTestParams(true))
+        // Open a new session
+        val aliceSession2 = commonTestHelper.logIntoAccount(aliceSession.myUserId, SessionTestParams(false))
+        // block key requesting for now as decrypt will send requests (room summary is trying to decrypt)
+        aliceSession2.cryptoService().enableKeyGossiping(false)
+        commonTestHelper.syncSession(aliceSession2)
+
+        Log.v("TEST", "=======> AliceSession 2 is ${aliceSession2.sessionParams.deviceId}")
 
         val roomSecondSessionPOV = aliceSession2.getRoom(roomId)
 
@@ -107,7 +100,10 @@ class KeyShareTests : InstrumentedTest {
         }
 
         val outgoingRequestsBefore = aliceSession2.cryptoService().getOutgoingRoomKeyRequests()
+        assertEquals("There should be no request as it's disabled", 0, outgoingRequestsBefore.size)
+
         // Try to request
+        aliceSession2.cryptoService().enableKeyGossiping(true)
         aliceSession2.cryptoService().requestRoomKeyForEvent(receivedEvent.root)
 
         val eventMegolmSessionId = receivedEvent.root.content.toModel<EncryptedEventContent>()?.sessionId
@@ -117,10 +113,6 @@ class KeyShareTests : InstrumentedTest {
         commonTestHelper.waitWithLatch { latch ->
             commonTestHelper.retryPeriodicallyWithLatch(latch) {
                 aliceSession2.cryptoService().getOutgoingRoomKeyRequests()
-                        .filter { req ->
-                            // filter out request that was known before
-                            !outgoingRequestsBefore.any { req.requestId == it.requestId }
-                        }
                         .let {
                             val outgoing = it.firstOrNull { it.sessionId == eventMegolmSessionId }
                             outGoingRequestId = outgoing?.requestId
@@ -141,20 +133,34 @@ class KeyShareTests : InstrumentedTest {
         commonTestHelper.waitWithLatch { latch ->
             commonTestHelper.retryPeriodicallyWithLatch(latch) {
                 // DEBUG LOGS
-                aliceSession.cryptoService().getIncomingRoomKeyRequests().let {
+//                aliceSession.cryptoService().getIncomingRoomKeyRequests().let {
-                    Log.v("TEST", "Incoming request Session 1 (looking for $outGoingRequestId)")
+//                    Log.v("TEST", "Incoming request Session 1 (looking for $outGoingRequestId)")
+//                    Log.v("TEST", "=========================")
+//                    it.forEach { keyRequest ->
+//                        Log.v("TEST", "[ts${keyRequest.localCreationTimestamp}] requestId ${keyRequest.requestId}, for sessionId ${keyRequest.requestBody?.sessionId}")
+//                    }
+//                    Log.v("TEST", "=========================")
+//                }
+
+                val incoming = aliceSession.cryptoService().getIncomingRoomKeyRequests().firstOrNull { it.requestId == outGoingRequestId }
+                incoming != null
+            }
+        }
+
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+                // DEBUG LOGS
+                aliceSession2.cryptoService().getOutgoingRoomKeyRequests().forEach { keyRequest ->
                     Log.v("TEST", "=========================")
-                    it.forEach { keyRequest ->
+                    Log.v("TEST", "requestId ${keyRequest.requestId}, for sessionId ${keyRequest.requestBody?.sessionId}")
-                        Log.v(
+                    Log.v("TEST", "replies -> ${keyRequest.results.joinToString { it.toString() }}")
-                                "TEST",
-                                "[ts${keyRequest.localCreationTimestamp}] requestId ${keyRequest.requestId}, for sessionId ${keyRequest.requestBody?.sessionId} is ${keyRequest.state}"
-                        )
-                    }
                     Log.v("TEST", "=========================")
                 }
 
-                val incoming = aliceSession.cryptoService().getIncomingRoomKeyRequests().firstOrNull { it.requestId == outGoingRequestId }
+                val outgoing = aliceSession2.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.requestId == outGoingRequestId }
-                incoming?.state == GossipingRequestState.REJECTED
+                val reply = outgoing?.results?.firstOrNull { it.userId == aliceSession.myUserId && it.fromDevice == aliceSession.sessionParams.deviceId }
+                val resultCode = (reply?.result as? RequestResult.Failure)?.code
+                resultCode == WithHeldCode.UNVERIFIED
             }
         }
 
@@ -175,254 +181,301 @@ class KeyShareTests : InstrumentedTest {
         // Re request
         aliceSession2.cryptoService().reRequestRoomKeyForEvent(receivedEvent.root)
 
-        commonTestHelper.waitWithLatch { latch ->
+        cryptoTestHelper.ensureCanDecrypt(listOf(receivedEvent.eventId), aliceSession2, roomId, listOf(sentEventText ?: ""))
-            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                aliceSession.cryptoService().getIncomingRoomKeyRequests().let {
-                    Log.v("TEST", "Incoming request Session 1")
-                    Log.v("TEST", "=========================")
-                    it.forEach {
-                        Log.v("TEST", "requestId ${it.requestId}, for sessionId ${it.requestBody?.sessionId} is ${it.state}")
-                    }
-                    Log.v("TEST", "=========================")
-
-                    it.any { it.requestBody?.sessionId == eventMegolmSessionId && it.state == GossipingRequestState.ACCEPTED }
-                }
-            }
-        }
-
-        Thread.sleep(6_000)
-        commonTestHelper.waitWithLatch { latch ->
-            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                aliceSession2.cryptoService().getOutgoingRoomKeyRequests().let {
-                    it.any { it.requestBody?.sessionId == eventMegolmSessionId && it.state == OutgoingGossipingRequestState.CANCELLED }
-                }
-            }
-        }
-
-        try {
-            commonTestHelper.runBlockingTest {
-                aliceSession2.cryptoService().decryptEvent(receivedEvent.root, "foo")
-            }
-        } catch (failure: Throwable) {
-            fail("should have been able to decrypt")
-        }
 
         commonTestHelper.signOutAndClose(aliceSession)
         commonTestHelper.signOutAndClose(aliceSession2)
     }
 
+    // See E2ESanityTest for a test regarding secret sharing
+
+    /**
+     * Test that the sender of a message accepts to re-share to another user
+     * if the key was originally shared with him
+     */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
+    fun test_reShareIfWasIntendedToBeShared() {
-    fun test_ShareSSSSSecret() {
+        val commonTestHelper = CommonTestHelper(context())
-        val aliceSession1 = commonTestHelper.createAccount(TestConstants.USER_ALICE, SessionTestParams(true))
+        val cryptoTestHelper = CryptoTestHelper(commonTestHelper)
 
-        commonTestHelper.doSync<Unit> {
+        val testData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
-            aliceSession1.cryptoService().crossSigningService()
+        val aliceSession = testData.firstSession
-                    .initializeCrossSigning(
+        val roomFromAlice = aliceSession.getRoom(testData.roomId)!!
-                            object : UserInteractiveAuthInterceptor {
+        val bobSession = testData.secondSession!!
-                                override fun performStage(flowResponse: RegistrationFlowResponse, errCode: String?, promise: Continuation<UIABaseAuth>) {
-                                    promise.resume(
-                                            UserPasswordAuth(
-                                                    user = aliceSession1.myUserId,
-                                                    password = TestConstants.PASSWORD
-                                            )
-                                    )
-                                }
-                            }, it
-                    )
-        }
 
-        // Also bootstrap keybackup on first session
+        val sentEvent = commonTestHelper.sendTextMessage(roomFromAlice, "Hello", 1).first()
-        val creationInfo = commonTestHelper.doSync<MegolmBackupCreationInfo> {
+        val sentEventMegolmSession = sentEvent.root.content.toModel<EncryptedEventContent>()!!.sessionId!!
-            aliceSession1.cryptoService().keysBackupService().prepareKeysBackupVersion(null, null, it)
-        }
-        val version = commonTestHelper.doSync<KeysVersion> {
-            aliceSession1.cryptoService().keysBackupService().createKeysBackupVersion(creationInfo, it)
-        }
-        // Save it for gossiping
-        aliceSession1.cryptoService().keysBackupService().saveBackupRecoveryKey(creationInfo.recoveryKey, version = version.version)
 
-        val aliceSession2 = commonTestHelper.logIntoAccount(aliceSession1.myUserId, SessionTestParams(true))
+        // bob should be able to decrypt
+        cryptoTestHelper.ensureCanDecrypt(listOf(sentEvent.eventId), bobSession, testData.roomId, listOf(sentEvent.getLastMessageContent()?.body ?: ""))
 
-        val aliceVerificationService1 = aliceSession1.cryptoService().verificationService()
+        // Let's try to request any how.
-        val aliceVerificationService2 = aliceSession2.cryptoService().verificationService()
+        // As it was share previously alice should accept to reshare
-
+        bobSession.cryptoService().reRequestRoomKeyForEvent(sentEvent.root)
-        // force keys download
-        commonTestHelper.doSync<MXUsersDevicesMap<CryptoDeviceInfo>> {
-            aliceSession1.cryptoService().downloadKeys(listOf(aliceSession1.myUserId), true, it)
-        }
-        commonTestHelper.doSync<MXUsersDevicesMap<CryptoDeviceInfo>> {
-            aliceSession2.cryptoService().downloadKeys(listOf(aliceSession2.myUserId), true, it)
-        }
-
-        var session1ShortCode: String? = null
-        var session2ShortCode: String? = null
-
-        aliceVerificationService1.addListener(object : VerificationService.Listener {
-            override fun transactionUpdated(tx: VerificationTransaction) {
-                Log.d("#TEST", "AA: tx incoming?:${tx.isIncoming} state ${tx.state}")
-                if (tx is SasVerificationTransaction) {
-                    if (tx.state == VerificationTxState.OnStarted) {
-                        (tx as IncomingSasVerificationTransaction).performAccept()
-                    }
-                    if (tx.state == VerificationTxState.ShortCodeReady) {
-                        session1ShortCode = tx.getDecimalCodeRepresentation()
-                        Thread.sleep(500)
-                        tx.userHasVerifiedShortCode()
-                    }
-                }
-            }
-        })
-
-        aliceVerificationService2.addListener(object : VerificationService.Listener {
-            override fun transactionUpdated(tx: VerificationTransaction) {
-                Log.d("#TEST", "BB: tx incoming?:${tx.isIncoming} state ${tx.state}")
-                if (tx is SasVerificationTransaction) {
-                    if (tx.state == VerificationTxState.ShortCodeReady) {
-                        session2ShortCode = tx.getDecimalCodeRepresentation()
-                        Thread.sleep(500)
-                        tx.userHasVerifiedShortCode()
-                    }
-                }
-            }
-        })
-
-        val txId = "m.testVerif12"
-        aliceVerificationService2.beginKeyVerification(
-                VerificationMethod.SAS, aliceSession1.myUserId, aliceSession1.sessionParams.deviceId
-                ?: "", txId
-        )
 
         commonTestHelper.waitWithLatch { latch ->
             commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                aliceSession1.cryptoService().getDeviceInfo(aliceSession1.myUserId, aliceSession2.sessionParams.deviceId ?: "")?.isVerified == true
+                val outgoing = bobSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+                val aliceReply = outgoing?.results?.firstOrNull { it.userId == aliceSession.myUserId && it.fromDevice == aliceSession.sessionParams.deviceId }
+                aliceReply != null && aliceReply.result is RequestResult.Success
             }
         }
-
-        assertNotNull(session1ShortCode)
-        Log.d("#TEST", "session1ShortCode: $session1ShortCode")
-        assertNotNull(session2ShortCode)
-        Log.d("#TEST", "session2ShortCode: $session2ShortCode")
-        assertEquals(session1ShortCode, session2ShortCode)
-
-        // SSK and USK private keys should have been shared
-
-        commonTestHelper.waitWithLatch(60_000) { latch ->
-            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                Log.d("#TEST", "CAN XS :${aliceSession2.cryptoService().crossSigningService().getMyCrossSigningKeys()}")
-                aliceSession2.cryptoService().crossSigningService().canCrossSign()
-            }
-        }
-
-        // Test that key backup key has been shared to
-        commonTestHelper.waitWithLatch(60_000) { latch ->
-            val keysBackupService = aliceSession2.cryptoService().keysBackupService()
-            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                Log.d("#TEST", "Recovery :${keysBackupService.getKeyBackupRecoveryKeyInfo()?.recoveryKey}")
-                keysBackupService.getKeyBackupRecoveryKeyInfo()?.recoveryKey == creationInfo.recoveryKey
-            }
-        }
-
-        commonTestHelper.signOutAndClose(aliceSession1)
-        commonTestHelper.signOutAndClose(aliceSession2)
     }
 
+    /**
+     * Test that our own devices accept to reshare to unverified device if it was shared initialy
+     * if the key was originally shared with him
+     */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
+    fun test_reShareToUnverifiedIfWasIntendedToBeShared() {
-    fun test_ImproperKeyShareBug() {
+        val commonTestHelper = CommonTestHelper(context())
-        val aliceSession = commonTestHelper.createAccount(TestConstants.USER_ALICE, SessionTestParams(true))
+        val cryptoTestHelper = CryptoTestHelper(commonTestHelper)
 
-        commonTestHelper.doSync<Unit> {
+        val testData = cryptoTestHelper.doE2ETestWithAliceInARoom(true)
-            aliceSession.cryptoService().crossSigningService()
+        val aliceSession = testData.firstSession
-                    .initializeCrossSigning(
+        val roomFromAlice = aliceSession.getRoom(testData.roomId)!!
-                            object : UserInteractiveAuthInterceptor {
+
-                                override fun performStage(flowResponse: RegistrationFlowResponse, errCode: String?, promise: Continuation<UIABaseAuth>) {
+        val aliceNewSession = commonTestHelper.logIntoAccount(aliceSession.myUserId, SessionTestParams(true))
-                                    promise.resume(
+
-                                            UserPasswordAuth(
+        // we wait for alice first session to be aware of that session?
-                                                    user = aliceSession.myUserId,
+        commonTestHelper.waitWithLatch { latch ->
-                                                    password = TestConstants.PASSWORD,
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                                                    session = flowResponse.session
+                val newSession = aliceSession.cryptoService().getUserDevices(aliceSession.myUserId)
-                                            )
+                        .firstOrNull { it.deviceId == aliceNewSession.sessionParams.deviceId }
-                                    )
+                newSession != null
-                                }
+            }
-                            }, it
-                    )
         }
+        val sentEvent = commonTestHelper.sendTextMessage(roomFromAlice, "Hello", 1).first()
+        val sentEventMegolmSession = sentEvent.root.content.toModel<EncryptedEventContent>()!!.sessionId!!
 
-        // Create an encrypted room and send a couple of messages
+        // Let's try to request any how.
-        val roomId = commonTestHelper.runBlockingTest {
+        // As it was share previously alice should accept to reshare
-            aliceSession.roomService().createRoom(
+        aliceNewSession.cryptoService().reRequestRoomKeyForEvent(sentEvent.root)
-                    CreateRoomParams().apply {
+
-                        visibility = RoomDirectoryVisibility.PRIVATE
+        commonTestHelper.waitWithLatch { latch ->
-                        enableEncryption()
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-                    }
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
-            )
+                val ownDeviceReply =
+                        outgoing?.results?.firstOrNull { it.userId == aliceSession.myUserId && it.fromDevice == aliceSession.sessionParams.deviceId }
+                ownDeviceReply != null && ownDeviceReply.result is RequestResult.Success
+            }
         }
-        val roomAlicePov = aliceSession.getRoom(roomId)
+    }
-        assertNotNull(roomAlicePov)
-        Thread.sleep(1_000)
-        assertTrue(roomAlicePov?.roomCryptoService()?.isEncrypted() == true)
-        val secondEventId = commonTestHelper.sendTextMessage(roomAlicePov!!, "Message", 3)[1].eventId
 
-        // Create bob session
+    /**
+     * Tests that keys reshared with own verified session are done from the earliest known index
+     */
+    @Test
+    fun test_reShareFromTheEarliestKnownIndexWithOwnVerifiedSession() {
+        val commonTestHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(commonTestHelper)
 
-        val bobSession = commonTestHelper.createAccount(TestConstants.USER_BOB, SessionTestParams(true))
+        val testData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
-        commonTestHelper.doSync<Unit> {
+        val aliceSession = testData.firstSession
-            bobSession.cryptoService().crossSigningService()
+        val bobSession = testData.secondSession!!
-                    .initializeCrossSigning(
+        val roomFromBob = bobSession.getRoom(testData.roomId)!!
-                            object : UserInteractiveAuthInterceptor {
-                                override fun performStage(flowResponse: RegistrationFlowResponse, errCode: String?, promise: Continuation<UIABaseAuth>) {
-                                    promise.resume(
-                                            UserPasswordAuth(
-                                                    user = bobSession.myUserId,
-                                                    password = TestConstants.PASSWORD,
-                                                    session = flowResponse.session
-                                            )
-                                    )
-                                }
-                            }, it
-                    )
-        }
 
-        // Let alice invite bob
+        val sentEvents = commonTestHelper.sendTextMessage(roomFromBob, "Hello", 3)
-        commonTestHelper.runBlockingTest {
+        val sentEventMegolmSession = sentEvents.first().root.content.toModel<EncryptedEventContent>()!!.sessionId!!
-            roomAlicePov.membershipService().invite(bobSession.myUserId, null)
-        }
 
-        commonTestHelper.runBlockingTest {
+        // Let alice now add a new session
-            bobSession.roomService().joinRoom(roomAlicePov.roomId, null, emptyList())
+        val aliceNewSession = commonTestHelper.logIntoAccount(aliceSession.myUserId, SessionTestParams(false))
-        }
+        aliceNewSession.cryptoService().enableKeyGossiping(false)
+        commonTestHelper.syncSession(aliceNewSession)
 
-        // we want to discard alice outbound session
+        // we wait bob first session to be aware of that session?
-        aliceSession.cryptoService().discardOutboundSession(roomAlicePov.roomId)
+        commonTestHelper.waitWithLatch { latch ->
-
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-        // and now resend a new message to reset index to 0
+                val newSession = bobSession.cryptoService().getUserDevices(aliceSession.myUserId)
-        commonTestHelper.sendTextMessage(roomAlicePov, "After", 1)
+                        .firstOrNull { it.deviceId == aliceNewSession.sessionParams.deviceId }
-
+                newSession != null
-        val roomRoomBobPov = aliceSession.getRoom(roomId)
-        val beforeJoin = roomRoomBobPov!!.getTimelineEvent(secondEventId)
-
-        var dRes = tryOrNull {
-            commonTestHelper.runBlockingTest {
-                bobSession.cryptoService().decryptEvent(beforeJoin!!.root, "")
             }
         }
 
-        assert(dRes == null)
+        val newEvent = commonTestHelper.sendTextMessage(roomFromBob, "The New", 1).first()
+        val newEventId = newEvent.eventId
+        val newEventText = newEvent.getLastMessageContent()!!.body
 
-        // Try to re-ask the keys
+        // alice should be able to decrypt the new one
+        cryptoTestHelper.ensureCanDecrypt(listOf(newEventId), aliceNewSession, testData.roomId, listOf(newEventText))
+        // but not the first one!
+        cryptoTestHelper.ensureCannotDecrypt(sentEvents.map { it.eventId }, aliceNewSession, testData.roomId)
 
-        bobSession.cryptoService().reRequestRoomKeyForEvent(beforeJoin!!.root)
+        // All should be using the same session id
+        sentEvents.forEach {
+            assertEquals(sentEventMegolmSession, it.root.content.toModel<EncryptedEventContent>()!!.sessionId)
+        }
+        assertEquals(sentEventMegolmSession, newEvent.root.content.toModel<EncryptedEventContent>()!!.sessionId)
 
-        Thread.sleep(3_000)
+        // Request a first time, bob should reply with unauthorized and alice should reply with unverified
+        aliceNewSession.cryptoService().enableKeyGossiping(true)
+        aliceNewSession.cryptoService().reRequestRoomKeyForEvent(newEvent.root)
 
-        // With the bug the first session would have improperly reshare that key :/
+        commonTestHelper.waitWithLatch { latch ->
-        dRes = tryOrNull {
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
-            commonTestHelper.runBlockingTest {
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
-                bobSession.cryptoService().decryptEvent(beforeJoin.root, "")
+                val ownDeviceReply = outgoing?.results
+                        ?.firstOrNull { it.userId == aliceSession.myUserId && it.fromDevice == aliceSession.sessionParams.deviceId }
+                val result = ownDeviceReply?.result
+                Log.v("TEST", "own device result is $result")
+                result != null && result is RequestResult.Failure && result.code == WithHeldCode.UNVERIFIED
             }
         }
-        Log.d("#TEST", "KS: sgould not decrypt that ${beforeJoin.root.getClearContent().toModel<MessageContent>()?.body}")
+
-        assert(dRes?.clearEvent == null)
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+                val bobDeviceReply = outgoing?.results
+                        ?.firstOrNull { it.userId == bobSession.myUserId && it.fromDevice == bobSession.sessionParams.deviceId }
+                val result = bobDeviceReply?.result
+                Log.v("TEST", "bob device result is $result")
+                result != null && result is RequestResult.Success && result.chainIndex > 0
+            }
+        }
+
+        // it's a success but still can't decrypt first message
+        cryptoTestHelper.ensureCannotDecrypt(sentEvents.map { it.eventId }, aliceNewSession, testData.roomId)
+
+        // Mark the new session as verified
+        aliceSession.cryptoService()
+                .verificationService()
+                .markedLocallyAsManuallyVerified(aliceNewSession.myUserId, aliceNewSession.sessionParams.deviceId!!)
+
+        // Let's now try to request
+        aliceNewSession.cryptoService().reRequestRoomKeyForEvent(sentEvents.first().root)
+
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+                // DEBUG LOGS
+                aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().forEach { keyRequest ->
+                    Log.v("TEST", "=========================")
+                    Log.v("TEST", "requestId ${keyRequest.requestId}, for sessionId ${keyRequest.requestBody?.sessionId}")
+                    Log.v("TEST", "replies -> ${keyRequest.results.joinToString { it.toString() }}")
+                    Log.v("TEST", "=========================")
+                }
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+                val ownDeviceReply =
+                        outgoing?.results?.firstOrNull { it.userId == aliceSession.myUserId && it.fromDevice == aliceSession.sessionParams.deviceId }
+                val result = ownDeviceReply?.result
+                result != null && result is RequestResult.Success && result.chainIndex == 0
+            }
+        }
+
+        // now the new session should be able to decrypt all!
+        cryptoTestHelper.ensureCanDecrypt(
+                sentEvents.map { it.eventId },
+                aliceNewSession,
+                testData.roomId,
+                sentEvents.map { it.getLastMessageContent()!!.body }
+        )
+
+        // Additional test, can we check that bob replied successfully but with a ratcheted key
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+                val bobReply = outgoing?.results?.firstOrNull { it.userId == bobSession.myUserId }
+                val result = bobReply?.result
+                result != null && result is RequestResult.Success && result.chainIndex == 3
+            }
+        }
+
+        commonTestHelper.signOutAndClose(aliceNewSession)
+        commonTestHelper.signOutAndClose(aliceSession)
+        commonTestHelper.signOutAndClose(bobSession)
+    }
+
+    /**
+     * Tests that we don't cancel a request to early on first forward if the index is not good enough
+     */
+    @Test
+    fun test_dontCancelToEarly() {
+        val commonTestHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(commonTestHelper)
+
+        val testData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom(true)
+        val aliceSession = testData.firstSession
+        val bobSession = testData.secondSession!!
+        val roomFromBob = bobSession.getRoom(testData.roomId)!!
+
+        val sentEvents = commonTestHelper.sendTextMessage(roomFromBob, "Hello", 3)
+        val sentEventMegolmSession = sentEvents.first().root.content.toModel<EncryptedEventContent>()!!.sessionId!!
+
+        // Let alice now add a new session
+        val aliceNewSession = commonTestHelper.logIntoAccount(aliceSession.myUserId, SessionTestParams(true))
+
+        // we wait bob first session to be aware of that session?
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+                val newSession = bobSession.cryptoService().getUserDevices(aliceSession.myUserId)
+                        .firstOrNull { it.deviceId == aliceNewSession.sessionParams.deviceId }
+                newSession != null
+            }
+        }
+
+        val newEvent = commonTestHelper.sendTextMessage(roomFromBob, "The New", 1).first()
+        val newEventId = newEvent.eventId
+        val newEventText = newEvent.getLastMessageContent()!!.body
+
+        // alice should be able to decrypt the new one
+        cryptoTestHelper.ensureCanDecrypt(listOf(newEventId), aliceNewSession, testData.roomId, listOf(newEventText))
+        // but not the first one!
+        cryptoTestHelper.ensureCannotDecrypt(sentEvents.map { it.eventId }, aliceNewSession, testData.roomId)
+
+        // All should be using the same session id
+        sentEvents.forEach {
+            assertEquals(sentEventMegolmSession, it.root.content.toModel<EncryptedEventContent>()!!.sessionId)
+        }
+        assertEquals(sentEventMegolmSession, newEvent.root.content.toModel<EncryptedEventContent>()!!.sessionId)
+
+        // Mark the new session as verified
+        aliceSession.cryptoService()
+                .verificationService()
+                .markedLocallyAsManuallyVerified(aliceNewSession.myUserId, aliceNewSession.sessionParams.deviceId!!)
+
+        // /!\ Stop initial alice session syncing so that it can't reply
+        aliceSession.cryptoService().enableKeyGossiping(false)
+        aliceSession.stopSync()
+
+        // Let's now try to request
+        aliceNewSession.cryptoService().reRequestRoomKeyForEvent(sentEvents.first().root)
+
+        // Should get a reply from bob and not from alice
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+              //  Log.d("#TEST", "outgoing key requests :${aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().joinToString { it.sessionId ?: "?" }}")
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+                val bobReply = outgoing?.results?.firstOrNull { it.userId == bobSession.myUserId }
+                val result = bobReply?.result
+                result != null && result is RequestResult.Success && result.chainIndex == 3
+            }
+        }
+
+        val outgoingReq = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+
+        assertNull("We should not have a reply from first session", outgoingReq!!.results.firstOrNull { it.fromDevice == aliceSession.sessionParams.deviceId })
+        assertEquals("The request should not be canceled", OutgoingRoomKeyRequestState.SENT, outgoingReq.state)
+
+        // let's wake up alice
+        aliceSession.cryptoService().enableKeyGossiping(true)
+        aliceSession.startSync(true)
+
+        // We should now get a reply from first session
+        commonTestHelper.waitWithLatch { latch ->
+            commonTestHelper.retryPeriodicallyWithLatch(latch) {
+                val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+                val ownDeviceReply =
+                        outgoing?.results?.firstOrNull { it.userId == aliceSession.myUserId && it.fromDevice == aliceSession.sessionParams.deviceId }
+                val result = ownDeviceReply?.result
+                result != null && result is RequestResult.Success && result.chainIndex == 0
+            }
+        }
+
+        // It should be in sent then cancel
+        val outgoing = aliceNewSession.cryptoService().getOutgoingRoomKeyRequests().firstOrNull { it.sessionId == sentEventMegolmSession }
+        assertEquals("The request should be canceled", OutgoingRoomKeyRequestState.SENT_THEN_CANCELED, outgoing!!.state)
+
+        commonTestHelper.signOutAndClose(aliceNewSession)
+        commonTestHelper.signOutAndClose(aliceSession)
+        commonTestHelper.signOutAndClose(bobSession)
     }
 }

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/gossiping/WithHeldTests.kt

@@ -21,7 +21,6 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
 import org.junit.Assert
 import org.junit.FixMethodOrder
-import org.junit.Ignore
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.MethodSorters
@@ -29,6 +28,7 @@ import org.matrix.android.sdk.InstrumentedTest
 import org.matrix.android.sdk.api.NoOpMatrixCallback
 import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
+import org.matrix.android.sdk.api.session.crypto.RequestResult
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
 import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
@@ -46,12 +46,11 @@ import org.matrix.android.sdk.common.TestConstants
 @LargeTest
 class WithHeldTests : InstrumentedTest {
 
-    private val testHelper = CommonTestHelper(context())
-    private val cryptoTestHelper = CryptoTestHelper(testHelper)
-
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun test_WithHeldUnverifiedReason() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         // =============================
         // ARRANGE
         // =============================
@@ -69,7 +68,6 @@ class WithHeldTests : InstrumentedTest {
         val roomAlicePOV = aliceSession.getRoom(roomId)!!
 
         val bobUnverifiedSession = testHelper.logIntoAccount(bobSession.myUserId, SessionTestParams(true))
-
         // =============================
         // ACT
         // =============================
@@ -88,6 +86,7 @@ class WithHeldTests : InstrumentedTest {
 
         val eventBobPOV = bobUnverifiedSession.getRoom(roomId)?.getTimelineEvent(timelineEvent.eventId)!!
 
+        val megolmSessionId = eventBobPOV.root.content.toModel<EncryptedEventContent>()!!.sessionId!!
         // =============================
         // ASSERT
         // =============================
@@ -103,9 +102,23 @@ class WithHeldTests : InstrumentedTest {
             val type = (failure as MXCryptoError.Base).errorType
             val technicalMessage = failure.technicalMessage
             Assert.assertEquals("Error should be withheld", MXCryptoError.ErrorType.KEYS_WITHHELD, type)
-            Assert.assertEquals("Cause should be unverified", WithHeldCode.UNVERIFIED.value, technicalMessage)
+            Assert.assertEquals("Cause should be unverified", WithHeldCode.UNAUTHORISED.value, technicalMessage)
         }
 
+        // Let's see if the reply we got from bob first session is unverified
+        testHelper.waitWithLatch { latch ->
+            testHelper.retryPeriodicallyWithLatch(latch) {
+                bobUnverifiedSession.cryptoService().getOutgoingRoomKeyRequests()
+                        .firstOrNull { it.sessionId == megolmSessionId }
+                        ?.results
+                        ?.firstOrNull { it.fromDevice == bobSession.sessionParams.deviceId }
+                        ?.result
+                        ?.let {
+                            it as? RequestResult.Failure
+                        }
+                        ?.code == WithHeldCode.UNVERIFIED
+            }
+        }
         // enable back sending to unverified
         aliceSession.cryptoService().setGlobalBlacklistUnverifiedDevices(false)
 
@@ -130,7 +143,7 @@ class WithHeldTests : InstrumentedTest {
             val type = (failure as MXCryptoError.Base).errorType
             val technicalMessage = failure.technicalMessage
             Assert.assertEquals("Error should be withheld", MXCryptoError.ErrorType.KEYS_WITHHELD, type)
-            Assert.assertEquals("Cause should be unverified", WithHeldCode.UNVERIFIED.value, technicalMessage)
+            Assert.assertEquals("Cause should be unverified", WithHeldCode.UNAUTHORISED.value, technicalMessage)
         }
 
         testHelper.signOutAndClose(aliceSession)
@@ -139,8 +152,10 @@ class WithHeldTests : InstrumentedTest {
     }
 
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun test_WithHeldNoOlm() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val testData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom()
         val aliceSession = testData.firstSession
         val bobSession = testData.secondSession!!
@@ -220,8 +235,10 @@ class WithHeldTests : InstrumentedTest {
     }
 
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun test_WithHeldKeyRequest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val testData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoom()
         val aliceSession = testData.firstSession
         val bobSession = testData.secondSession!!
@@ -267,5 +284,8 @@ class WithHeldTests : InstrumentedTest {
                 wc?.code == WithHeldCode.UNAUTHORISED
             }
         }
+
+        testHelper.signOutAndClose(aliceSession)
+        testHelper.signOutAndClose(bobSecondSession)
     }
 }

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTest.kt

@@ -24,7 +24,6 @@ import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertNull
 import org.junit.Assert.assertTrue
 import org.junit.FixMethodOrder
-import org.junit.Ignore
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.MethodSorters
@@ -37,7 +36,9 @@ import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupLastVersio
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupState
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupStateListener
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupVersionTrust
+import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupVersionTrustSignature
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersion
+import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersionResult
 import org.matrix.android.sdk.api.session.crypto.keysbackup.MegolmBackupCreationInfo
 import org.matrix.android.sdk.api.session.crypto.keysbackup.toKeysVersionResult
 import org.matrix.android.sdk.api.session.crypto.model.ImportRoomKeysResult
@@ -54,18 +55,16 @@ import java.util.concurrent.CountDownLatch
 @LargeTest
 class KeysBackupTest : InstrumentedTest {
 
-    private val testHelper = CommonTestHelper(context())
-    private val cryptoTestHelper = CryptoTestHelper(testHelper)
-    private val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
-
     /**
      * - From doE2ETestWithAliceAndBobInARoomWithEncryptedMessages, we should have no backed up keys
      * - Check backup keys after having marked one as backed up
      * - Reset keys backup markers
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun roomKeysTest_testBackupStore_ok() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
         // From doE2ETestWithAliceAndBobInARoomWithEncryptedMessages, we should have no backed up keys
@@ -104,6 +103,8 @@ class KeysBackupTest : InstrumentedTest {
      */
     @Test
     fun prepareKeysBackupVersionTest() {
+        val testHelper = CommonTestHelper(context())
+
         val bobSession = testHelper.createAccount(TestConstants.USER_BOB, KeysBackupTestConstants.defaultSessionParams)
 
         assertNotNull(bobSession.cryptoService().keysBackupService())
@@ -132,7 +133,11 @@ class KeysBackupTest : InstrumentedTest {
      */
     @Test
     fun createKeysBackupVersionTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+
         val bobSession = testHelper.createAccount(TestConstants.USER_BOB, KeysBackupTestConstants.defaultSessionParams)
+        cryptoTestHelper.initializeCrossSigning(bobSession)
 
         val keysBackup = bobSession.cryptoService().keysBackupService()
 
@@ -147,13 +152,46 @@ class KeysBackupTest : InstrumentedTest {
         assertFalse(keysBackup.isEnabled)
 
         // Create the version
-        testHelper.doSync<KeysVersion> {
+        val version = testHelper.doSync<KeysVersion> {
             keysBackup.createKeysBackupVersion(megolmBackupCreationInfo, it)
         }
 
         // Backup must be enable now
         assertTrue(keysBackup.isEnabled)
 
+        // Check that it's signed with MSK
+        val versionResult = testHelper.doSync<KeysVersionResult?> {
+            keysBackup.getVersion(version.version, it)
+        }
+        val trust = testHelper.doSync<KeysBackupVersionTrust> {
+            keysBackup.getKeysBackupTrust(versionResult!!, it)
+        }
+
+        assertEquals("Should have 2 signatures", 2, trust.signatures.size)
+
+        trust.signatures
+                .firstOrNull { it is KeysBackupVersionTrustSignature.DeviceSignature }
+                .let {
+                    assertNotNull("Should be signed by a device", it)
+                    it as KeysBackupVersionTrustSignature.DeviceSignature
+                }.let {
+                    assertEquals("Should be signed by current device", bobSession.sessionParams.deviceId, it.deviceId)
+                    assertTrue("Signature should be valid", it.valid)
+                }
+
+        trust.signatures
+                .firstOrNull { it is KeysBackupVersionTrustSignature.UserSignature }
+                .let {
+                    assertNotNull("Should be signed by a user", it)
+                    it as KeysBackupVersionTrustSignature.UserSignature
+                }.let {
+                    val msk = bobSession.cryptoService().crossSigningService()
+                            .getMyCrossSigningKeys()?.masterKey()?.unpaddedBase64PublicKey
+                    assertEquals("Should be signed by my msk 1", msk, it.keyId)
+                    assertEquals("Should be signed by my msk 2", msk, it.cryptoCrossSigningKey?.unpaddedBase64PublicKey)
+                    assertTrue("Signature should be valid", it.valid)
+                }
+
         stateObserver.stopAndCheckStates(null)
         testHelper.signOutAndClose(bobSession)
     }
@@ -163,8 +201,11 @@ class KeysBackupTest : InstrumentedTest {
      * - Check the backup completes
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun backupAfterCreateKeysBackupVersionTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
         keysBackupTestHelper.waitForKeybackUpBatching()
@@ -204,8 +245,11 @@ class KeysBackupTest : InstrumentedTest {
      * Check that backupAllGroupSessions() returns valid data
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun backupAllGroupSessionsTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
         val keysBackup = cryptoTestData.firstSession.cryptoService().keysBackupService()
@@ -249,8 +293,11 @@ class KeysBackupTest : InstrumentedTest {
      * - Compare the decrypted megolm key with the original one
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun testEncryptAndDecryptKeysBackupData() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
         val keysBackup = cryptoTestData.firstSession.cryptoService().keysBackupService() as DefaultKeysBackupService
@@ -293,8 +340,11 @@ class KeysBackupTest : InstrumentedTest {
      * - Restore must be successful
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun restoreKeysBackupTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(null)
 
         // - Restore the e2e backup from the homeserver
@@ -378,8 +428,11 @@ class KeysBackupTest : InstrumentedTest {
      * - It must be trusted and must have with 2 signatures now
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun trustKeyBackupVersionTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Do an e2e backup to the homeserver with a recovery key
         // - And log Alice on a new device
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(null)
@@ -438,8 +491,11 @@ class KeysBackupTest : InstrumentedTest {
      * - It must be trusted and must have with 2 signatures now
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun trustKeyBackupVersionWithRecoveryKeyTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Do an e2e backup to the homeserver with a recovery key
         // - And log Alice on a new device
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(null)
@@ -496,8 +552,11 @@ class KeysBackupTest : InstrumentedTest {
      * - The backup must still be untrusted and disabled
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun trustKeyBackupVersionWithWrongRecoveryKeyTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Do an e2e backup to the homeserver with a recovery key
         // - And log Alice on a new device
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(null)
@@ -538,8 +597,11 @@ class KeysBackupTest : InstrumentedTest {
      * - It must be trusted and must have with 2 signatures now
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun trustKeyBackupVersionWithPasswordTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val password = "Password"
 
         // - Do an e2e backup to the homeserver with a password
@@ -598,8 +660,11 @@ class KeysBackupTest : InstrumentedTest {
      * - The backup must still be untrusted and disabled
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun trustKeyBackupVersionWithWrongPasswordTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val password = "Password"
         val badPassword = "Bad Password"
 
@@ -639,8 +704,11 @@ class KeysBackupTest : InstrumentedTest {
      * - It must fail
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun restoreKeysBackupWithAWrongRecoveryKeyTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(null)
 
         // - Try to restore the e2e backup with a wrong recovery key
@@ -673,8 +741,11 @@ class KeysBackupTest : InstrumentedTest {
      * - Restore must be successful
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun testBackupWithPassword() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val password = "password"
 
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(password)
@@ -730,8 +801,11 @@ class KeysBackupTest : InstrumentedTest {
      * - It must fail
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun restoreKeysBackupWithAWrongPasswordTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val password = "password"
         val wrongPassword = "passw0rd"
 
@@ -767,8 +841,11 @@ class KeysBackupTest : InstrumentedTest {
      * - Restore must be successful
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun testUseRecoveryKeyToRestoreAPasswordBasedKeysBackup() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val password = "password"
 
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(password)
@@ -797,8 +874,11 @@ class KeysBackupTest : InstrumentedTest {
      * - It must fail
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun testUsePasswordToRestoreARecoveryKeyBasedKeysBackup() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         val testData = keysBackupTestHelper.createKeysBackupScenarioWithPassword(null)
 
         // - Try to restore the e2e backup with a password
@@ -829,8 +909,11 @@ class KeysBackupTest : InstrumentedTest {
      * - Check the returned KeysVersionResult is trusted
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun testIsKeysBackupTrusted() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Create a backup version
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
@@ -855,7 +938,7 @@ class KeysBackupTest : InstrumentedTest {
         assertTrue(keysBackupVersionTrust.usable)
         assertEquals(1, keysBackupVersionTrust.signatures.size)
 
-        val signature = keysBackupVersionTrust.signatures[0]
+        val signature = keysBackupVersionTrust.signatures[0] as KeysBackupVersionTrustSignature.DeviceSignature
         assertTrue(signature.valid)
         assertNotNull(signature.device)
         assertEquals(cryptoTestData.firstSession.cryptoService().getMyDevice().deviceId, signature.deviceId)
@@ -865,66 +948,6 @@ class KeysBackupTest : InstrumentedTest {
         cryptoTestData.cleanUp(testHelper)
     }
 
-    /**
-     * Check backup starts automatically if there is an existing and compatible backup
-     * version on the homeserver.
-     * - Create a backup version
-     * - Restart alice session
-     * -> The new alice session must back up to the same version
-     */
-    @Test
-    @Ignore("This test will be ignored until it is fixed")
-    fun testCheckAndStartKeysBackupWhenRestartingAMatrixSession() {
-        // - Create a backup version
-        val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
-
-        val keysBackup = cryptoTestData.firstSession.cryptoService().keysBackupService()
-
-        val stateObserver = StateObserver(keysBackup)
-
-        assertFalse(keysBackup.isEnabled)
-
-        val keyBackupCreationInfo = keysBackupTestHelper.prepareAndCreateKeysBackupData(keysBackup)
-
-        assertTrue(keysBackup.isEnabled)
-
-        // - Restart alice session
-        // - Log Alice on a new device
-        val aliceSession2 = testHelper.logIntoAccount(cryptoTestData.firstSession.myUserId, KeysBackupTestConstants.defaultSessionParamsWithInitialSync)
-
-        cryptoTestData.cleanUp(testHelper)
-
-        val keysBackup2 = aliceSession2.cryptoService().keysBackupService()
-
-        val stateObserver2 = StateObserver(keysBackup2)
-
-        // -> The new alice session must back up to the same version
-        val latch = CountDownLatch(1)
-        var count = 0
-        keysBackup2.addListener(object : KeysBackupStateListener {
-            override fun onStateChange(newState: KeysBackupState) {
-                // Check the backup completes
-                if (newState == KeysBackupState.ReadyToBackUp) {
-                    count++
-
-                    if (count == 2) {
-                        // Remove itself from the list of listeners
-                        keysBackup2.removeListener(this)
-
-                        latch.countDown()
-                    }
-                }
-            }
-        })
-        testHelper.await(latch)
-
-        assertEquals(keyBackupCreationInfo.version, keysBackup2.currentBackupVersion)
-
-        stateObserver.stopAndCheckStates(null)
-        stateObserver2.stopAndCheckStates(null)
-        testHelper.signOutAndClose(aliceSession2)
-    }
-
     /**
      * Check WrongBackUpVersion state
      *
@@ -935,6 +958,10 @@ class KeysBackupTest : InstrumentedTest {
      */
     @Test
     fun testBackupWhenAnotherBackupWasCreated() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Create a backup version
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
@@ -1005,8 +1032,11 @@ class KeysBackupTest : InstrumentedTest {
      * -> It must success
      */
     @Test
-    @Ignore("This test will be ignored until it is fixed")
     fun testBackupAfterVerifyingADevice() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Create a backup version
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 
@@ -1039,6 +1069,8 @@ class KeysBackupTest : InstrumentedTest {
         // - Try to backup all in aliceSession2, it must fail
         val keysBackup2 = aliceSession2.cryptoService().keysBackupService()
 
+        assertFalse("Backup should not be enabled", keysBackup2.isEnabled)
+
         val stateObserver2 = StateObserver(keysBackup2)
 
         var isSuccessful = false
@@ -1056,8 +1088,8 @@ class KeysBackupTest : InstrumentedTest {
         assertFalse(isSuccessful)
 
         // Backup state must be NotTrusted
-        assertEquals(KeysBackupState.NotTrusted, keysBackup2.state)
+        assertEquals("Backup state must be NotTrusted", KeysBackupState.NotTrusted, keysBackup2.state)
-        assertFalse(keysBackup2.isEnabled)
+        assertFalse("Backup should not be enabled", keysBackup2.isEnabled)
 
         // - Validate the old device from the new one
         aliceSession2.cryptoService().setDeviceVerification(
@@ -1103,6 +1135,10 @@ class KeysBackupTest : InstrumentedTest {
      */
     @Test
     fun deleteKeysBackupTest() {
+        val testHelper = CommonTestHelper(context())
+        val cryptoTestHelper = CryptoTestHelper(testHelper)
+        val keysBackupTestHelper = KeysBackupTestHelper(testHelper, cryptoTestHelper)
+
         // - Create a backup version
         val cryptoTestData = cryptoTestHelper.doE2ETestWithAliceAndBobInARoomWithEncryptedMessages()
 

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/keysbackup/KeysBackupTestHelper.kt

@@ -106,14 +106,14 @@ internal class KeysBackupTestHelper(
 
         Assert.assertNotNull(megolmBackupCreationInfo)
 
-        Assert.assertFalse(keysBackup.isEnabled)
+        Assert.assertFalse("Key backup should not be enabled before creation", keysBackup.isEnabled)
 
         // Create the version
         val keysVersion = testHelper.doSync<KeysVersion> {
             keysBackup.createKeysBackupVersion(megolmBackupCreationInfo, it)
         }
 
-        Assert.assertNotNull(keysVersion.version)
+        Assert.assertNotNull("Key backup version should not be null", keysVersion.version)
 
         // Backup must be enable now
         Assert.assertTrue(keysBackup.isEnabled)

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/crypto/verification/qrcode/VerificationTest.kt

@@ -19,7 +19,6 @@ package org.matrix.android.sdk.internal.crypto.verification.qrcode
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.amshove.kluent.shouldBe
 import org.junit.FixMethodOrder
-import org.junit.Ignore
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.MethodSorters
@@ -40,7 +39,6 @@ import kotlin.coroutines.resume
 
 @RunWith(AndroidJUnit4::class)
 @FixMethodOrder(MethodSorters.JVM)
-@Ignore("This test is flaky ; see issue #5449")
 class VerificationTest : InstrumentedTest {
 
     data class ExpectedResult(

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/crypto/MXCryptoConfig.kt

@@ -31,5 +31,11 @@ data class MXCryptoConfig constructor(
          * If set to false, the request will be forwarded to the application layer; in this
          * case the application can decide to prompt the user.
          */
-        val discardRoomKeyRequestsFromUntrustedDevices: Boolean = true
+        val discardRoomKeyRequestsFromUntrustedDevices: Boolean = true,
+
+        /**
+         * Currently megolm keys are requested to the sender device and to all of our devices.
+         * You can limit request only to your sessions by turning this setting to `true`
+         */
+        val limitRoomKeyRequestsToMyDevices: Boolean = false,
 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/logger/LoggerTag.kt

@@ -22,15 +22,15 @@ package org.matrix.android.sdk.api.logger
  * val loggerTag = LoggerTag("MyTag", LoggerTag.VOIP)
  * Timber.tag(loggerTag.value).v("My log message")
  */
-open class LoggerTag(_value: String, parentTag: LoggerTag? = null) {
+open class LoggerTag(name: String, parentTag: LoggerTag? = null) {
 
     object SYNC : LoggerTag("SYNC")
     object VOIP : LoggerTag("VOIP")
     object CRYPTO : LoggerTag("CRYPTO")
 
     val value: String = if (parentTag == null) {
-        _value
+        name
     } else {
-        "${parentTag.value}/$_value"
+        "${parentTag.value}/$name"
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/CryptoService.kt

@@ -26,6 +26,7 @@ import org.matrix.android.sdk.api.session.crypto.crosssigning.CrossSigningServic
 import org.matrix.android.sdk.api.session.crypto.crosssigning.DeviceTrustLevel
 import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysBackupService
 import org.matrix.android.sdk.api.session.crypto.keyshare.GossipingRequestListener
+import org.matrix.android.sdk.api.session.crypto.model.AuditTrail
 import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.DeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.DevicesListResponse
@@ -35,8 +36,6 @@ import org.matrix.android.sdk.api.session.crypto.model.MXDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.MXEncryptEventContentResult
 import org.matrix.android.sdk.api.session.crypto.model.MXEventDecryptionResult
 import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingRoomKeyRequest
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
 import org.matrix.android.sdk.api.session.crypto.verification.VerificationService
 import org.matrix.android.sdk.api.session.events.model.Content
 import org.matrix.android.sdk.api.session.events.model.Event
@@ -76,6 +75,15 @@ interface CryptoService {
 
     fun setGlobalBlacklistUnverifiedDevices(block: Boolean)
 
+    /**
+     * Enable or disable key gossiping.
+     * Default is true.
+     * If set to false this device won't send key_request nor will accept key forwarded
+     */
+    fun enableKeyGossiping(enable: Boolean)
+
+    fun isKeyGossipingEnabled(): Boolean
+
     fun setRoomUnBlacklistUnverifiedDevices(roomId: String)
 
     fun getDeviceTrackingStatus(userId: String): Int
@@ -94,8 +102,6 @@ interface CryptoService {
 
     fun reRequestRoomKeyForEvent(event: Event)
 
-    fun cancelRoomKeyRequest(requestBody: RoomKeyRequestBody)
-
     fun addRoomKeysRequestListener(listener: GossipingRequestListener)
 
     fun removeRoomKeysRequestListener(listener: GossipingRequestListener)
@@ -142,14 +148,20 @@ interface CryptoService {
     fun addNewSessionListener(newSessionListener: NewSessionListener)
     fun removeSessionListener(listener: NewSessionListener)
 
-    fun getOutgoingRoomKeyRequests(): List<OutgoingRoomKeyRequest>
+    fun getOutgoingRoomKeyRequests(): List<OutgoingKeyRequest>
-    fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>>
+    fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingKeyRequest>>
 
     fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
     fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>>
 
-    fun getGossipingEventsTrail(): LiveData<PagedList<Event>>
+    /**
-    fun getGossipingEvents(): List<Event>
+     * Can be called by the app layer to accept a request manually
+     * Use carefully as it is prone to social attacks
+     */
+    suspend fun manuallyAcceptRoomKeyRequest(request: IncomingRoomKeyRequest)
+
+    fun getGossipingEventsTrail(): LiveData<PagedList<AuditTrail>>
+    fun getGossipingEvents(): List<AuditTrail>
 
     // For testing shared session
     fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int>

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/OutgoingKeyRequest.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.session.crypto
+
+import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
+import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
+
+data class RequestReply(
+        val userId: String,
+        val fromDevice: String?,
+        val result: RequestResult
+)
+
+sealed class RequestResult {
+    data class  Success(val chainIndex: Int) : RequestResult()
+    data class Failure(val code: WithHeldCode) : RequestResult()
+}
+
+data class OutgoingKeyRequest(
+        var requestBody: RoomKeyRequestBody?,
+        // recipients for the request map of users to list of deviceId
+        val recipients: Map<String, List<String>>,
+        val fromIndex: Int,
+        // Unique id for this request. Used for both
+        // an id within the request for later pairing with a cancellation, and for
+        // the transaction id when sending the to_device messages to our local
+        val requestId: String, // current state of this request
+        val state: OutgoingRoomKeyRequestState,
+        val results: List<RequestReply>
+) {
+    /**
+     * Used only for log.
+     *
+     * @return the room id.
+     */
+    val roomId = requestBody?.roomId
+
+    /**
+     * Used only for log.
+     *
+     * @return the session id
+     */
+    val sessionId = requestBody?.sessionId
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/OutgoingRoomKeyRequestState.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
+ * Copyright (c) 2022 The Matrix.org Foundation C.I.C.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,14 +14,20 @@
  * limitations under the License.
  */
 
-package org.matrix.android.sdk.api.session.crypto.model
+package org.matrix.android.sdk.api.session.crypto
 
-enum class OutgoingGossipingRequestState {
+enum class OutgoingRoomKeyRequestState {
     UNSENT,
-    SENDING,
     SENT,
-    CANCELLING,
+    SENT_THEN_CANCELED,
-    CANCELLED,
+    CANCELLATION_PENDING,
-    FAILED_TO_SEND,
+    CANCELLATION_PENDING_AND_WILL_RESEND;
-    FAILED_TO_CANCEL
+
+    companion object {
+        fun pendingStates() = setOf(
+                UNSENT,
+                CANCELLATION_PENDING_AND_WILL_RESEND,
+                CANCELLATION_PENDING
+        )
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/keysbackup/KeysBackupVersionTrustSignature.kt

@@ -16,25 +16,35 @@
 
 package org.matrix.android.sdk.api.session.crypto.keysbackup
 
+import org.matrix.android.sdk.api.session.crypto.crosssigning.CryptoCrossSigningKey
 import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 
 /**
  * A signature in a `KeysBackupVersionTrust` object.
  */
-data class KeysBackupVersionTrustSignature(
-        /**
-         * The id of the device that signed the backup version.
-         */
-        val deviceId: String?,
 
-        /**
+sealed class KeysBackupVersionTrustSignature {
-         * The device that signed the backup version.
-         * Can be null if the device is not known.
-         */
-        val device: CryptoDeviceInfo?,
 
-        /**
+    data class DeviceSignature(
-         * Flag to indicate the signature from this device is valid.
+            /**
-         */
+             * The id of the device that signed the backup version.
-        val valid: Boolean,
+             */
-)
+            val deviceId: String?,
+
+            /**
+             * The device that signed the backup version.
+             * Can be null if the device is not known.
+             */
+            val device: CryptoDeviceInfo?,
+
+            /**
+             * Flag to indicate the signature from this device is valid.
+             */
+            val valid: Boolean) : KeysBackupVersionTrustSignature()
+
+    data class UserSignature(
+            val keyId: String?,
+            val cryptoCrossSigningKey: CryptoCrossSigningKey?,
+            val valid: Boolean
+    ) : KeysBackupVersionTrustSignature()
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/keyshare/GossipingRequestListener.kt

@@ -16,9 +16,8 @@
 
 package org.matrix.android.sdk.api.session.crypto.keyshare
 
-import org.matrix.android.sdk.api.session.crypto.model.IncomingRequestCancellation
 import org.matrix.android.sdk.api.session.crypto.model.IncomingRoomKeyRequest
-import org.matrix.android.sdk.api.session.crypto.model.IncomingSecretShareRequest
+import org.matrix.android.sdk.api.session.crypto.model.SecretShareRequest
 
 /**
  * Room keys events listener
@@ -35,12 +34,12 @@ interface GossipingRequestListener {
      * Returns the secret value to be shared
      * @return true if is handled
      */
-    fun onSecretShareRequest(request: IncomingSecretShareRequest): Boolean
+    fun onSecretShareRequest(request: SecretShareRequest): Boolean
 
     /**
      * A room key request cancellation has been received.
      *
      * @param request the cancellation request
      */
-    fun onRoomKeyRequestCancellation(request: IncomingRequestCancellation)
+    fun onRequestCancelled(request: IncomingRoomKeyRequest)
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/model/AuditTrail.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.session.crypto.model
+
+import com.squareup.moshi.JsonClass
+import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
+
+enum class TrailType {
+    OutgoingKeyForward,
+    IncomingKeyForward,
+    OutgoingKeyWithheld,
+    IncomingKeyRequest,
+    Unknown
+}
+
+interface AuditInfo {
+    val roomId: String
+    val sessionId: String
+    val senderKey: String
+    val alg: String
+    val userId: String
+    val deviceId: String
+}
+
+@JsonClass(generateAdapter = true)
+data class ForwardInfo(
+        override val roomId: String,
+        override val sessionId: String,
+        override val senderKey: String,
+        override val alg: String,
+        override val userId: String,
+        override val deviceId: String,
+        val chainIndex: Long?
+) : AuditInfo
+
+object UnknownInfo : AuditInfo {
+    override val roomId: String = ""
+    override val sessionId: String = ""
+    override val senderKey: String = ""
+    override val alg: String = ""
+    override val userId: String = ""
+    override val deviceId: String = ""
+}
+
+@JsonClass(generateAdapter = true)
+data class WithheldInfo(
+        override val roomId: String,
+        override val sessionId: String,
+        override val senderKey: String,
+        override val alg: String,
+        val code: WithHeldCode,
+        override val userId: String,
+        override val deviceId: String
+) : AuditInfo
+
+@JsonClass(generateAdapter = true)
+data class IncomingKeyRequestInfo(
+        override val roomId: String,
+        override val sessionId: String,
+        override val senderKey: String,
+        override val alg: String,
+        override val userId: String,
+        override val deviceId: String,
+        val requestId: String
+) : AuditInfo
+
+data class AuditTrail(
+        val ageLocalTs: Long,
+        val type: TrailType,
+        val info: AuditInfo
+)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/model/IncomingRequestCancellation.kt

@@ -1,64 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.api.session.crypto.model
-
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.internal.crypto.IncomingShareRequestCommon
-import org.matrix.android.sdk.internal.crypto.model.rest.ShareRequestCancellation
-
-/**
- * IncomingRequestCancellation describes the incoming room key cancellation.
- */
-data class IncomingRequestCancellation(
-        /**
-         * The user id
-         */
-        override val userId: String? = null,
-
-        /**
-         * The device id
-         */
-        override val deviceId: String? = null,
-
-        /**
-         * The request id
-         */
-        override val requestId: String? = null,
-        override val localCreationTimestamp: Long?
-) : IncomingShareRequestCommon {
-    companion object {
-        /**
-         * Factory
-         *
-         * @param event the event
-         * @param currentTimeMillis the current time in milliseconds
-         */
-        fun fromEvent(event: Event, currentTimeMillis: Long): IncomingRequestCancellation? {
-            return event.getClearContent()
-                    .toModel<ShareRequestCancellation>()
-                    ?.let {
-                        IncomingRequestCancellation(
-                                userId = event.senderId,
-                                deviceId = it.requestingDeviceId,
-                                requestId = it.requestId,
-                                localCreationTimestamp = event.ageLocalTs ?: currentTimeMillis
-                        )
-                    }
-        }
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/model/IncomingRoomKeyRequest.kt

@@ -16,9 +16,7 @@
 
 package org.matrix.android.sdk.api.session.crypto.model
 
-import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.internal.util.time.Clock
-import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.internal.crypto.IncomingShareRequestCommon
 
 /**
  * IncomingRoomKeyRequest class defines the incoming room keys request.
@@ -27,38 +25,25 @@ data class IncomingRoomKeyRequest(
         /**
          * The user id
          */
-        override val userId: String? = null,
+        val userId: String? = null,
 
         /**
          * The device id
          */
-        override val deviceId: String? = null,
+        val deviceId: String? = null,
 
         /**
          * The request id
          */
-        override val requestId: String? = null,
+        val requestId: String? = null,
 
         /**
          * The request body
          */
         val requestBody: RoomKeyRequestBody? = null,
 
-        val state: GossipingRequestState = GossipingRequestState.NONE,
+        val localCreationTimestamp: Long?
-
+) {
-        /**
-         * The runnable to call to accept to share the keys
-         */
-        @Transient
-        var share: Runnable? = null,
-
-        /**
-         * The runnable to call to ignore the key share request.
-         */
-        @Transient
-        var ignore: Runnable? = null,
-        override val localCreationTimestamp: Long?
-) : IncomingShareRequestCommon {
     companion object {
         /**
          * Factory
@@ -66,18 +51,36 @@ data class IncomingRoomKeyRequest(
          * @param event the event
          * @param currentTimeMillis the current time in milliseconds
          */
-        fun fromEvent(event: Event, currentTimeMillis: Long): IncomingRoomKeyRequest? {
+        fun fromEvent(trail: AuditTrail): IncomingRoomKeyRequest? {
-            return event.getClearContent()
+            return trail
-                    .toModel<RoomKeyShareRequest>()
+                    .takeIf { it.type == TrailType.IncomingKeyRequest }
+                    ?.let {
+                        it.info as? IncomingKeyRequestInfo
+                    }
                     ?.let {
                         IncomingRoomKeyRequest(
-                                userId = event.senderId,
+                                userId = it.userId,
-                                deviceId = it.requestingDeviceId,
+                                deviceId = it.deviceId,
                                 requestId = it.requestId,
-                                requestBody = it.body ?: RoomKeyRequestBody(),
+                                requestBody = RoomKeyRequestBody(
-                                localCreationTimestamp = event.ageLocalTs ?: currentTimeMillis
+                                        algorithm = it.alg,
+                                        roomId = it.roomId,
+                                        senderKey = it.senderKey,
+                                        sessionId = it.sessionId
+                                ),
+                                localCreationTimestamp = trail.ageLocalTs
                         )
                     }
         }
+
+        internal fun fromRestRequest(senderId: String, request: RoomKeyShareRequest, clock: Clock): IncomingRoomKeyRequest? {
+            return IncomingRoomKeyRequest(
+                    userId = senderId,
+                    deviceId = request.requestingDeviceId,
+                    requestId = request.requestId,
+                    requestBody = request.body,
+                    localCreationTimestamp = clock.epochMillis()
+            )
+        }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/model/IncomingSecretShareRequest.kt

@@ -1,83 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.api.session.crypto.model
-
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.internal.crypto.IncomingShareRequestCommon
-
-/**
- * IncomingSecretShareRequest class defines the incoming secret keys request.
- */
-data class IncomingSecretShareRequest(
-        /**
-         * The user id
-         */
-        override val userId: String? = null,
-
-        /**
-         * The device id
-         */
-        override val deviceId: String? = null,
-
-        /**
-         * The request id
-         */
-        override val requestId: String? = null,
-
-        /**
-         * The request body
-         */
-        val secretName: String? = null,
-
-        /**
-         * The runnable to call to accept to share the keys
-         */
-        @Transient
-        var share: ((String) -> Unit)? = null,
-
-        /**
-         * The runnable to call to ignore the key share request.
-         */
-        @Transient
-        var ignore: Runnable? = null,
-
-        override val localCreationTimestamp: Long?
-
-) : IncomingShareRequestCommon {
-    companion object {
-        /**
-         * Factory
-         *
-         * @param event the event
-         * @param currentTimeMillis the current time in milliseconds
-         */
-        fun fromEvent(event: Event, currentTimeMillis: Long): IncomingSecretShareRequest? {
-            return event.getClearContent()
-                    .toModel<SecretShareRequest>()
-                    ?.let {
-                        IncomingSecretShareRequest(
-                                userId = event.senderId,
-                                deviceId = it.requestingDeviceId,
-                                requestId = it.requestId,
-                                secretName = it.secretName,
-                                localCreationTimestamp = event.ageLocalTs ?: currentTimeMillis
-                        )
-                    }
-        }
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/crypto/model/OutgoingRoomKeyRequest.kt

@@ -1,55 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.api.session.crypto.model
-
-import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequest
-
-/**
- * Represents an outgoing room key request
- */
-@JsonClass(generateAdapter = true)
-data class OutgoingRoomKeyRequest(
-        // RequestBody
-        val requestBody: RoomKeyRequestBody?,
-        // list of recipients for the request
-        override val recipients: Map<String, List<String>>,
-        // Unique id for this request. Used for both
-        // an id within the request for later pairing with a cancellation, and for
-        // the transaction id when sending the to_device messages to our local
-        override val requestId: String, // current state of this request
-        override val state: OutgoingGossipingRequestState
-        // transaction id for the cancellation, if any
-        // override var cancellationTxnId: String? = null
-) : OutgoingGossipingRequest {
-
-    /**
-     * Used only for log.
-     *
-     * @return the room id.
-     */
-    val roomId: String?
-        get() = requestBody?.roomId
-
-    /**
-     * Used only for log.
-     *
-     * @return the session id
-     */
-    val sessionId: String?
-        get() = requestBody?.sessionId
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/content/RoomKeyWithHeldContent.kt

@@ -52,7 +52,13 @@ data class RoomKeyWithHeldContent(
         /**
          *  A human-readable reason for why the key was not sent. The receiving client should only use this string if it does not understand the code.
          */
-        @Json(name = "reason") val reason: String? = null
+        @Json(name = "reason") val reason: String? = null,
+
+        /**
+         * the device ID of the device sending the m.room_key.withheld message
+         * MSC3735
+         */
+        @Json(name = "from_device") val fromDevice: String? = null
 
 ) {
     val code: WithHeldCode?

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/presence/model/PresenceEnum.kt

@@ -28,7 +28,10 @@ enum class PresenceEnum(val value: String) {
     OFFLINE("offline"),
 
     @Json(name = "unavailable")
-    UNAVAILABLE("unavailable");
+    UNAVAILABLE("unavailable"),
+
+    @Json(name = "org.matrix.msc3026.busy")
+    BUSY("busy");
 
     companion object {
         fun from(s: String): PresenceEnum? = values().find { it.value == s }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/RoomGuestAccessContent.kt

@@ -27,13 +27,13 @@ import timber.log.Timber
 @JsonClass(generateAdapter = true)
 data class RoomGuestAccessContent(
         // Required. Whether guests can join the room. One of: ["can_join", "forbidden"]
-        @Json(name = "guest_access") val _guestAccess: String? = null
+        @Json(name = "guest_access") val guestAccessStr: String? = null
 ) {
-    val guestAccess: GuestAccess? = when (_guestAccess) {
+    val guestAccess: GuestAccess? = when (guestAccessStr) {
         "can_join"  -> GuestAccess.CanJoin
         "forbidden" -> GuestAccess.Forbidden
         else        -> {
-            Timber.w("Invalid value for GuestAccess: `$_guestAccess`")
+            Timber.w("Invalid value for GuestAccess: `$guestAccessStr`")
             null
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/RoomHistoryVisibilityContent.kt

@@ -22,15 +22,15 @@ import timber.log.Timber
 
 @JsonClass(generateAdapter = true)
 data class RoomHistoryVisibilityContent(
-        @Json(name = "history_visibility") val _historyVisibility: String? = null
+        @Json(name = "history_visibility") val historyVisibilityStr: String? = null
 ) {
-    val historyVisibility: RoomHistoryVisibility? = when (_historyVisibility) {
+    val historyVisibility: RoomHistoryVisibility? = when (historyVisibilityStr) {
         "world_readable" -> RoomHistoryVisibility.WORLD_READABLE
         "shared"         -> RoomHistoryVisibility.SHARED
         "invited"        -> RoomHistoryVisibility.INVITED
         "joined"         -> RoomHistoryVisibility.JOINED
         else             -> {
-            Timber.w("Invalid value for RoomHistoryVisibility: `$_historyVisibility`")
+            Timber.w("Invalid value for RoomHistoryVisibility: `$historyVisibilityStr`")
             null
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/RoomJoinRulesContent.kt

@@ -27,7 +27,7 @@ import timber.log.Timber
  */
 @JsonClass(generateAdapter = true)
 data class RoomJoinRulesContent(
-        @Json(name = "join_rule") val _joinRules: String? = null,
+        @Json(name = "join_rule") val joinRulesStr: String? = null,
         /**
          * If the allow key is an empty list (or not a list at all),
          * then no users are allowed to join without an invite.
@@ -35,14 +35,14 @@ data class RoomJoinRulesContent(
          */
         @Json(name = "allow") val allowList: List<RoomJoinRulesAllowEntry>? = null
 ) {
-    val joinRules: RoomJoinRules? = when (_joinRules) {
+    val joinRules: RoomJoinRules? = when (joinRulesStr) {
         "public"     -> RoomJoinRules.PUBLIC
         "invite"     -> RoomJoinRules.INVITE
         "knock"      -> RoomJoinRules.KNOCK
         "private"    -> RoomJoinRules.PRIVATE
         "restricted" -> RoomJoinRules.RESTRICTED
         else         -> {
-            Timber.w("Invalid value for RoomJoinRules: `$_joinRules`")
+            Timber.w("Invalid value for RoomJoinRules: `$joinRulesStr`")
             null
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/create/RoomFeaturePreset.kt

@@ -47,7 +47,7 @@ class RestrictedRoomPreset(val homeServerCapabilities: HomeServerCapabilities, v
                         type = EventType.STATE_ROOM_JOIN_RULES,
                         stateKey = "",
                         content = RoomJoinRulesContent(
-                                _joinRules = RoomJoinRules.RESTRICTED.value,
+                                joinRulesStr = RoomJoinRules.RESTRICTED.value,
                                 allowList = restrictedList
                         ).toContent()
                 )

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/timeline/TimelineSettings.kt

@@ -32,6 +32,10 @@ data class TimelineSettings(
          * The root thread eventId if this is a thread timeline, or null if this is NOT a thread timeline
          */
         val rootThreadEventId: String? = null,
+        /**
+         * If true Sender Info shown in room will get the latest data information (avatar + displayName)
+         */
+        val useLiveSenderInfo: Boolean = false,
 ) {
 
     /**

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/securestorage/SharedSecretStorageService.kt

@@ -131,7 +131,7 @@ interface SharedSecretStorageService {
 
     fun checkShouldBeAbleToAccessSecrets(secretNames: List<String>, keyId: String?): IntegrityResult
 
-    fun requestSecret(name: String, myOtherDeviceId: String)
+    suspend fun requestSecret(name: String, myOtherDeviceId: String)
 
     data class KeyRef(
             val keyId: String?,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/sync/model/LazyRoomSyncEphemeral.kt

@@ -20,6 +20,6 @@ import com.squareup.moshi.JsonClass
 
 @JsonClass(generateAdapter = false)
 sealed class LazyRoomSyncEphemeral {
-    data class Parsed(val _roomSyncEphemeral: RoomSyncEphemeral) : LazyRoomSyncEphemeral()
+    data class Parsed(val roomSyncEphemeral: RoomSyncEphemeral) : LazyRoomSyncEphemeral()
     object Stored : LazyRoomSyncEphemeral()
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/auth/DefaultAuthenticationService.kt

@@ -382,11 +382,16 @@ internal class DefaultAuthenticationService @Inject constructor(
         return getWellknownTask.execute(
                 GetWellknownTask.Params(
                         domain = matrixId.getDomain(),
-                        homeServerConnectionConfig = homeServerConnectionConfig
+                        homeServerConnectionConfig = homeServerConnectionConfig.orWellKnownDefaults()
                 )
         )
     }
 
+    private fun HomeServerConnectionConfig?.orWellKnownDefaults() = this ?: HomeServerConnectionConfig.Builder()
+            // server uri is ignored when doing a wellknown lookup as we use the matrix id domain instead
+            .withHomeServerUri("https://dummy.org")
+            .build()
+
     override suspend fun directAuthentication(homeServerConnectionConfig: HomeServerConnectionConfig,
                                               matrixId: String,
                                               password: String,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/auth/version/Versions.kt

@@ -74,8 +74,8 @@ internal fun Versions.isLoginAndRegistrationSupportedBySdk(): Boolean {
  * Indicate if the homeserver support MSC3440 for threads
  */
 internal fun Versions.doesServerSupportThreads(): Boolean {
-    return getMaxVersion() >= HomeServerVersion.v1_3_0 ||
+    // TODO Check for v1.3 or whichever spec version formally specifies MSC3440.
-            unstableFeatures?.get(FEATURE_THREADS_MSC3440_STABLE) ?: false
+    return unstableFeatures?.get(FEATURE_THREADS_MSC3440_STABLE) ?: false
 }
 
 /**

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/CancelGossipRequestWorker.kt

@@ -1,123 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import android.content.Context
-import androidx.work.WorkerParameters
-import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.api.auth.data.Credentials
-import org.matrix.android.sdk.api.failure.shouldBeRetried
-import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.toContent
-import org.matrix.android.sdk.internal.SessionManager
-import org.matrix.android.sdk.internal.crypto.model.rest.ShareRequestCancellation
-import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.crypto.tasks.createUniqueTxnId
-import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.util.time.Clock
-import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import javax.inject.Inject
-
-internal class CancelGossipRequestWorker(context: Context, params: WorkerParameters, sessionManager: SessionManager) :
-        SessionSafeCoroutineWorker<CancelGossipRequestWorker.Params>(context, params, sessionManager, Params::class.java) {
-
-    @JsonClass(generateAdapter = true)
-    internal data class Params(
-            override val sessionId: String,
-            val requestId: String,
-            val recipients: Map<String, List<String>>,
-            // The txnId for the sendToDevice request. Nullable for compatibility reasons, but MUST always be provided
-            // to use the same value if this worker is retried.
-            val txnId: String? = null,
-            override val lastFailureMessage: String? = null
-    ) : SessionWorkerParams {
-        companion object {
-            fun fromRequest(sessionId: String, request: OutgoingGossipingRequest): Params {
-                return Params(
-                        sessionId = sessionId,
-                        requestId = request.requestId,
-                        recipients = request.recipients,
-                        txnId = createUniqueTxnId(),
-                        lastFailureMessage = null
-                )
-            }
-        }
-    }
-
-    @Inject lateinit var sendToDeviceTask: SendToDeviceTask
-    @Inject lateinit var cryptoStore: IMXCryptoStore
-    @Inject lateinit var credentials: Credentials
-    @Inject lateinit var clock: Clock
-
-    override fun injectWith(injector: SessionComponent) {
-        injector.inject(this)
-    }
-
-    override suspend fun doSafeWork(params: Params): Result {
-        // params.txnId should be provided in all cases now. But Params can be deserialized by
-        // the WorkManager from data serialized in a previous version of the application, so without the txnId field.
-        // So if not present, we create a txnId
-        val txnId = params.txnId ?: createUniqueTxnId()
-        val contentMap = MXUsersDevicesMap<Any>()
-        val toDeviceContent = ShareRequestCancellation(
-                requestingDeviceId = credentials.deviceId,
-                requestId = params.requestId
-        )
-        cryptoStore.saveGossipingEvent(Event(
-                type = EventType.ROOM_KEY_REQUEST,
-                content = toDeviceContent.toContent(),
-                senderId = credentials.userId
-        ).also {
-            it.ageLocalTs = clock.epochMillis()
-        })
-
-        params.recipients.forEach { userToDeviceMap ->
-            userToDeviceMap.value.forEach { deviceId ->
-                contentMap.setObject(userToDeviceMap.key, deviceId, toDeviceContent)
-            }
-        }
-
-        try {
-            cryptoStore.updateOutgoingGossipingRequestState(params.requestId, OutgoingGossipingRequestState.CANCELLING)
-            sendToDeviceTask.execute(
-                    SendToDeviceTask.Params(
-                            eventType = EventType.ROOM_KEY_REQUEST,
-                            contentMap = contentMap,
-                            transactionId = txnId
-                    )
-            )
-            cryptoStore.updateOutgoingGossipingRequestState(params.requestId, OutgoingGossipingRequestState.CANCELLED)
-            return Result.success()
-        } catch (throwable: Throwable) {
-            return if (throwable.shouldBeRetried()) {
-                Result.retry()
-            } else {
-                cryptoStore.updateOutgoingGossipingRequestState(params.requestId, OutgoingGossipingRequestState.FAILED_TO_CANCEL)
-                buildErrorResult(params, throwable.localizedMessage ?: "error")
-            }
-        }
-    }
-
-    override fun buildErrorParams(params: Params, message: String): Params {
-        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DefaultCryptoService.kt

@@ -42,12 +42,14 @@ import org.matrix.android.sdk.api.logger.LoggerTag
 import org.matrix.android.sdk.api.session.crypto.CryptoService
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
 import org.matrix.android.sdk.api.session.crypto.NewSessionListener
+import org.matrix.android.sdk.api.session.crypto.OutgoingKeyRequest
 import org.matrix.android.sdk.api.session.crypto.crosssigning.DeviceTrustLevel
 import org.matrix.android.sdk.api.session.crypto.crosssigning.KEYBACKUP_SECRET_SSSS_NAME
 import org.matrix.android.sdk.api.session.crypto.crosssigning.MASTER_KEY_SSSS_NAME
 import org.matrix.android.sdk.api.session.crypto.crosssigning.SELF_SIGNING_KEY_SSSS_NAME
 import org.matrix.android.sdk.api.session.crypto.crosssigning.USER_SIGNING_KEY_SSSS_NAME
 import org.matrix.android.sdk.api.session.crypto.keyshare.GossipingRequestListener
+import org.matrix.android.sdk.api.session.crypto.model.AuditTrail
 import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.DeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.DevicesListResponse
@@ -57,15 +59,13 @@ import org.matrix.android.sdk.api.session.crypto.model.MXDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.MXEncryptEventContentResult
 import org.matrix.android.sdk.api.session.crypto.model.MXEventDecryptionResult
 import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingRoomKeyRequest
+import org.matrix.android.sdk.api.session.crypto.model.RoomKeyShareRequest
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
+import org.matrix.android.sdk.api.session.crypto.model.TrailType
 import org.matrix.android.sdk.api.session.events.model.Content
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
 import org.matrix.android.sdk.api.session.events.model.content.RoomKeyContent
 import org.matrix.android.sdk.api.session.events.model.content.RoomKeyWithHeldContent
-import org.matrix.android.sdk.api.session.events.model.content.SecretSendEventContent
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.room.model.Membership
 import org.matrix.android.sdk.api.session.room.model.RoomHistoryVisibility
@@ -76,7 +76,6 @@ import org.matrix.android.sdk.internal.crypto.actions.MegolmSessionDataImporter
 import org.matrix.android.sdk.internal.crypto.actions.SetDeviceVerificationAction
 import org.matrix.android.sdk.internal.crypto.algorithms.IMXEncrypting
 import org.matrix.android.sdk.internal.crypto.algorithms.IMXGroupEncryption
-import org.matrix.android.sdk.internal.crypto.algorithms.IMXWithHeldExtension
 import org.matrix.android.sdk.internal.crypto.algorithms.megolm.MXMegolmEncryptionFactory
 import org.matrix.android.sdk.internal.crypto.algorithms.olm.MXOlmEncryptionFactory
 import org.matrix.android.sdk.internal.crypto.crosssigning.DefaultCrossSigningService
@@ -91,6 +90,7 @@ import org.matrix.android.sdk.internal.crypto.tasks.GetDevicesTask
 import org.matrix.android.sdk.internal.crypto.tasks.SetDeviceNameTask
 import org.matrix.android.sdk.internal.crypto.tasks.UploadKeysTask
 import org.matrix.android.sdk.internal.crypto.verification.DefaultVerificationService
+import org.matrix.android.sdk.internal.crypto.verification.VerificationMessageProcessor
 import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.MoshiProvider
 import org.matrix.android.sdk.internal.di.UserId
@@ -156,9 +156,10 @@ internal class DefaultCryptoService @Inject constructor(
 
         private val crossSigningService: DefaultCrossSigningService,
         //
-        private val incomingGossipingRequestManager: IncomingGossipingRequestManager,
+        private val incomingKeyRequestManager: IncomingKeyRequestManager,
+        private val secretShareManager: SecretShareManager,
         //
-        private val outgoingGossipingRequestManager: OutgoingGossipingRequestManager,
+        private val outgoingKeyRequestManager: OutgoingKeyRequestManager,
         // Actions
         private val setDeviceVerificationAction: SetDeviceVerificationAction,
         private val megolmSessionDataImporter: MegolmSessionDataImporter,
@@ -178,6 +179,7 @@ internal class DefaultCryptoService @Inject constructor(
         private val taskExecutor: TaskExecutor,
         private val cryptoCoroutineScope: CoroutineScope,
         private val eventDecryptor: EventDecryptor,
+        private val verificationMessageProcessor: VerificationMessageProcessor,
         private val liveEventManager: Lazy<StreamEventsManager>
 ) : CryptoService {
 
@@ -192,7 +194,7 @@ internal class DefaultCryptoService @Inject constructor(
         }
     }
 
-    fun onLiveEvent(roomId: String, event: Event) {
+    fun onLiveEvent(roomId: String, event: Event, isInitialSync: Boolean) {
         // handle state events
         if (event.isStateEvent()) {
             when (event.type) {
@@ -201,9 +203,18 @@ internal class DefaultCryptoService @Inject constructor(
                 EventType.STATE_ROOM_HISTORY_VISIBILITY -> onRoomHistoryVisibilityEvent(roomId, event)
             }
         }
+
+        // handle verification
+        if (!isInitialSync) {
+            if (event.type != null && verificationMessageProcessor.shouldProcess(event.type)) {
+                cryptoCoroutineScope.launch(coroutineDispatchers.dmVerif) {
+                    verificationMessageProcessor.process(event)
+                }
+            }
+        }
     }
 
-    val gossipingBuffer = mutableListOf<Event>()
+//    val gossipingBuffer = mutableListOf<Event>()
 
     override fun setDeviceName(deviceId: String, deviceName: String, callback: MatrixCallback<Unit>) {
         setDeviceNameTask
@@ -379,27 +390,8 @@ internal class DefaultCryptoService @Inject constructor(
         // Open the store
         cryptoStore.open()
 
-        runCatching {
+        isStarting.set(false)
-//            if (isInitialSync) {
+        isStarted.set(true)
-//                // refresh the devices list for each known room members
-//                deviceListManager.invalidateAllDeviceLists()
-//                deviceListManager.refreshOutdatedDeviceLists()
-//            } else {
-
-            // Why would we do that? it will be called at end of syn
-            incomingGossipingRequestManager.processReceivedGossipingRequests()
-//            }
-        }.fold(
-                {
-                    isStarting.set(false)
-                    isStarted.set(true)
-                },
-                {
-                    isStarting.set(false)
-                    isStarted.set(false)
-                    Timber.tag(loggerTag.value).e(it, "Start failed")
-                }
-        )
     }
 
     /**
@@ -407,7 +399,8 @@ internal class DefaultCryptoService @Inject constructor(
      */
     fun close() = runBlocking(coroutineDispatchers.crypto) {
         cryptoCoroutineScope.coroutineContext.cancelChildren(CancellationException("Closing crypto module"))
-        incomingGossipingRequestManager.close()
+        incomingKeyRequestManager.close()
+        outgoingKeyRequestManager.close()
         olmDevice.release()
         cryptoStore.close()
     }
@@ -472,15 +465,28 @@ internal class DefaultCryptoService @Inject constructor(
                     }
 
                     oneTimeKeysUploader.maybeUploadOneTimeKeys()
-                    incomingGossipingRequestManager.processReceivedGossipingRequests()
                 }
-            }
 
-            tryOrNull {
+                // Process pending key requests
-                gossipingBuffer.toList().let {
+                try {
-                    cryptoStore.saveGossipingEvents(it)
+                    if (toDevices.isEmpty()) {
+                        // this is not blocking
+                        outgoingKeyRequestManager.requireProcessAllPendingKeyRequests()
+                    } else {
+                        Timber.tag(loggerTag.value)
+                                .w("Don't process key requests yet as there might be more to_device to catchup")
+                    }
+                } catch (failure: Throwable) {
+                    // just for safety but should not throw
+                    Timber.tag(loggerTag.value).w("failed to process pending request")
+                }
+
+                try {
+                    incomingKeyRequestManager.processIncomingRequests()
+                } catch (failure: Throwable) {
+                    // just for safety but should not throw
+                    Timber.tag(loggerTag.value).w("failed to process incoming room key requests")
                 }
-                gossipingBuffer.clear()
             }
         }
     }
@@ -594,7 +600,7 @@ internal class DefaultCryptoService @Inject constructor(
         // (for now at least. Maybe we should alert the user somehow?)
         val existingAlgorithm = cryptoStore.getRoomAlgorithm(roomId)
 
-        if (existingAlgorithm == algorithm && roomEncryptorsStore.get(roomId) != null) {
+        if (existingAlgorithm == algorithm) {
             // ignore
             Timber.tag(loggerTag.value).e("setEncryptionInRoom() : Ignoring m.room.encryption for same alg ($algorithm) in  $roomId")
             return false
@@ -787,19 +793,25 @@ internal class DefaultCryptoService @Inject constructor(
         cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
             when (event.getClearType()) {
                 EventType.ROOM_KEY, EventType.FORWARDED_ROOM_KEY -> {
-                    gossipingBuffer.add(event)
                     // Keys are imported directly, not waiting for end of sync
                     onRoomKeyEvent(event)
                 }
-                EventType.REQUEST_SECRET,
+                EventType.REQUEST_SECRET                         -> {
+                    secretShareManager.handleSecretRequest(event)
+                }
                 EventType.ROOM_KEY_REQUEST                       -> {
-                    // save audit trail
+                    event.getClearContent().toModel<RoomKeyShareRequest>()?.let { req ->
-                    gossipingBuffer.add(event)
+                        // We'll always get these because we send room key requests to
-                    // Requests are stacked, and will be handled one by one at the end of the sync (onSyncComplete)
+                        // '*' (ie. 'all devices') which includes the sending device,
-                    incomingGossipingRequestManager.onGossipingRequestEvent(event)
+                        // so ignore requests from ourself because apart from it being
+                        // very silly, it won't work because an Olm session cannot send
+                        // messages to itself.
+                        if (req.requestingDeviceId != deviceId) { // ignore self requests
+                            event.senderId?.let { incomingKeyRequestManager.addNewIncomingRequest(it, req) }
+                        }
+                    }
                 }
                 EventType.SEND_SECRET                            -> {
-                    gossipingBuffer.add(event)
                     onSecretSendReceived(event)
                 }
                 EventType.ROOM_KEY_WITHHELD                      -> {
@@ -837,50 +849,38 @@ internal class DefaultCryptoService @Inject constructor(
         val withHeldContent = event.getClearContent().toModel<RoomKeyWithHeldContent>() ?: return Unit.also {
             Timber.tag(loggerTag.value).i("Malformed onKeyWithHeldReceived() : missing fields")
         }
-        Timber.tag(loggerTag.value).i("onKeyWithHeldReceived() received from:${event.senderId}, content <$withHeldContent>")
+        val senderId = event.senderId ?: return Unit.also {
-        val alg = roomDecryptorProvider.getOrCreateRoomDecryptor(withHeldContent.roomId, withHeldContent.algorithm)
+            Timber.tag(loggerTag.value).i("Malformed onKeyWithHeldReceived() : missing fields")
-        if (alg is IMXWithHeldExtension) {
-            alg.onRoomKeyWithHeldEvent(withHeldContent)
-        } else {
-            Timber.tag(loggerTag.value).e("onKeyWithHeldReceived() from:${event.senderId}: Unable to handle WithHeldContent for ${withHeldContent.algorithm}")
-            return
         }
+        withHeldContent.sessionId ?: return
+        withHeldContent.algorithm ?: return
+        withHeldContent.roomId ?: return
+        withHeldContent.senderKey ?: return
+        outgoingKeyRequestManager.onRoomKeyWithHeld(
+                sessionId = withHeldContent.sessionId,
+                algorithm = withHeldContent.algorithm,
+                roomId = withHeldContent.roomId,
+                senderKey = withHeldContent.senderKey,
+                fromDevice = withHeldContent.fromDevice,
+                event = Event(
+                        type = EventType.ROOM_KEY_WITHHELD,
+                        senderId = senderId,
+                        content = event.getClearContent()
+                )
+        )
     }
 
-    private fun onSecretSendReceived(event: Event) {
+    private suspend fun onSecretSendReceived(event: Event) {
-        Timber.tag(loggerTag.value).i("GOSSIP onSecretSend() from ${event.senderId} : onSecretSendReceived ${event.content?.get("sender_key")}")
+        secretShareManager.onSecretSendReceived(event) { secretName, secretValue ->
-        if (!event.isEncrypted()) {
+            handleSDKLevelGossip(secretName, secretValue)
-            // secret send messages must be encrypted
-            Timber.tag(loggerTag.value).e("GOSSIP onSecretSend() :Received unencrypted secret send event")
-            return
-        }
-
-        // Was that sent by us?
-        if (event.senderId != userId) {
-            Timber.tag(loggerTag.value).e("GOSSIP onSecretSend() : Ignore secret from other user ${event.senderId}")
-            return
-        }
-
-        val secretContent = event.getClearContent().toModel<SecretSendEventContent>() ?: return
-
-        val existingRequest = cryptoStore
-                .getOutgoingSecretKeyRequests().firstOrNull { it.requestId == secretContent.requestId }
-
-        if (existingRequest == null) {
-            Timber.tag(loggerTag.value).i("GOSSIP onSecretSend() : Ignore secret that was not requested: ${secretContent.requestId}")
-            return
-        }
-
-        if (!handleSDKLevelGossip(existingRequest.secretName, secretContent.secretValue)) {
-            // TODO Ask to application layer?
-            Timber.tag(loggerTag.value).v("onSecretSend() : secret not handled by SDK")
         }
     }
 
     /**
      * Returns true if handled by SDK, otherwise should be sent to application layer
      */
-    private fun handleSDKLevelGossip(secretName: String?, secretValue: String): Boolean {
+    private fun handleSDKLevelGossip(secretName: String?,
+                                     secretValue: String): Boolean {
         return when (secretName) {
             MASTER_KEY_SSSS_NAME       -> {
                 crossSigningService.onSecretMSKGossip(secretValue)
@@ -1095,6 +1095,12 @@ internal class DefaultCryptoService @Inject constructor(
         cryptoStore.setGlobalBlacklistUnverifiedDevices(block)
     }
 
+    override fun enableKeyGossiping(enable: Boolean) {
+        cryptoStore.enableKeyGossiping(enable)
+    }
+
+    override fun isKeyGossipingEnabled() = cryptoStore.isKeyGossipingEnabled()
+
     /**
      * Tells whether the client should ever send encrypted messages to unverified devices.
      * The default value is false.
@@ -1158,52 +1164,17 @@ internal class DefaultCryptoService @Inject constructor(
         setRoomBlacklistUnverifiedDevices(roomId, false)
     }
 
-// TODO Check if this method is still necessary
-    /**
-     * Cancel any earlier room key request
-     *
-     * @param requestBody requestBody
-     */
-    override fun cancelRoomKeyRequest(requestBody: RoomKeyRequestBody) {
-        outgoingGossipingRequestManager.cancelRoomKeyRequest(requestBody)
-    }
-
     /**
      * Re request the encryption keys required to decrypt an event.
      *
      * @param event the event to decrypt again.
      */
     override fun reRequestRoomKeyForEvent(event: Event) {
-        val wireContent = event.content.toModel<EncryptedEventContent>() ?: return Unit.also {
+        outgoingKeyRequestManager.requestKeyForEvent(event, true)
-            Timber.tag(loggerTag.value).e("reRequestRoomKeyForEvent Failed to re-request key, null content")
-        }
-
-        val requestBody = RoomKeyRequestBody(
-                algorithm = wireContent.algorithm,
-                roomId = event.roomId,
-                senderKey = wireContent.senderKey,
-                sessionId = wireContent.sessionId
-        )
-
-        outgoingGossipingRequestManager.resendRoomKeyRequest(requestBody)
     }
 
     override fun requestRoomKeyForEvent(event: Event) {
-        val wireContent = event.content.toModel<EncryptedEventContent>() ?: return Unit.also {
+        outgoingKeyRequestManager.requestKeyForEvent(event, false)
-            Timber.tag(loggerTag.value).e("requestRoomKeyForEvent Failed to request key, null content eventId: ${event.eventId}")
-        }
-
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-//            if (!isStarted()) {
-//                Timber.v("## CRYPTO | requestRoomKeyForEvent() : wait after e2e init")
-//                internalStart(false)
-//            }
-            roomDecryptorProvider
-                    .getOrCreateRoomDecryptor(event.roomId, wireContent.algorithm)
-                    ?.requestKeysForEvent(event, false) ?: run {
-                Timber.tag(loggerTag.value).v("requestRoomKeyForEvent() : No room decryptor for roomId:${event.roomId} algorithm:${wireContent.algorithm}")
-            }
-        }
     }
 
     /**
@@ -1212,7 +1183,8 @@ internal class DefaultCryptoService @Inject constructor(
      * @param listener listener
      */
     override fun addRoomKeysRequestListener(listener: GossipingRequestListener) {
-        incomingGossipingRequestManager.addRoomKeysRequestListener(listener)
+        incomingKeyRequestManager.addRoomKeysRequestListener(listener)
+        secretShareManager.addListener(listener)
     }
 
     /**
@@ -1221,42 +1193,10 @@ internal class DefaultCryptoService @Inject constructor(
      * @param listener listener
      */
     override fun removeRoomKeysRequestListener(listener: GossipingRequestListener) {
-        incomingGossipingRequestManager.removeRoomKeysRequestListener(listener)
+        incomingKeyRequestManager.removeRoomKeysRequestListener(listener)
+        secretShareManager.removeListener(listener)
     }
 
-//    private fun markOlmSessionForUnwedging(senderId: String, deviceInfo: CryptoDeviceInfo) {
-//        val deviceKey = deviceInfo.identityKey()
-//
-//        val lastForcedDate = lastNewSessionForcedDates.getObject(senderId, deviceKey) ?: 0
-//        val now = clock.epochMillis()
-//        if (now - lastForcedDate < CRYPTO_MIN_FORCE_SESSION_PERIOD_MILLIS) {
-//            Timber.d("## CRYPTO | markOlmSessionForUnwedging: New session already forced with device at $lastForcedDate. Not forcing another")
-//            return
-//        }
-//
-//        Timber.d("## CRYPTO | markOlmSessionForUnwedging from $senderId:${deviceInfo.deviceId}")
-//        lastNewSessionForcedDates.setObject(senderId, deviceKey, now)
-//
-//        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-//            ensureOlmSessionsForDevicesAction.handle(mapOf(senderId to listOf(deviceInfo)), force = true)
-//
-//            // Now send a blank message on that session so the other side knows about it.
-//            // (The keyshare request is sent in the clear so that won't do)
-//            // We send this first such that, as long as the toDevice messages arrive in the
-//            // same order we sent them, the other end will get this first, set up the new session,
-//            // then get the keyshare request and send the key over this new session (because it
-//            // is the session it has most recently received a message on).
-//            val payloadJson = mapOf<String, Any>("type" to EventType.DUMMY)
-//
-//            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
-//            val sendToDeviceMap = MXUsersDevicesMap<Any>()
-//            sendToDeviceMap.setObject(senderId, deviceInfo.deviceId, encodedPayload)
-//            Timber.v("## CRYPTO | markOlmSessionForUnwedging() : sending to $senderId:${deviceInfo.deviceId}")
-//            val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
-//            sendToDeviceTask.execute(sendToDeviceParams)
-//        }
-//    }
-
     /**
      * Provides the list of unknown devices
      *
@@ -1302,27 +1242,41 @@ internal class DefaultCryptoService @Inject constructor(
         return "DefaultCryptoService of $userId ($deviceId)"
     }
 
-    override fun getOutgoingRoomKeyRequests(): List<OutgoingRoomKeyRequest> {
+    override fun getOutgoingRoomKeyRequests(): List<OutgoingKeyRequest> {
         return cryptoStore.getOutgoingRoomKeyRequests()
     }
 
-    override fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>> {
+    override fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingKeyRequest>> {
         return cryptoStore.getOutgoingRoomKeyRequestsPaged()
     }
 
-    override fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>> {
-        return cryptoStore.getIncomingRoomKeyRequestsPaged()
-    }
-
     override fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest> {
-        return cryptoStore.getIncomingRoomKeyRequests()
+        return cryptoStore.getGossipingEvents()
+                .mapNotNull {
+                    IncomingRoomKeyRequest.fromEvent(it)
+                }
     }
 
-    override fun getGossipingEventsTrail(): LiveData<PagedList<Event>> {
+    override fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>> {
+        return cryptoStore.getGossipingEventsTrail(TrailType.IncomingKeyRequest) {
+            IncomingRoomKeyRequest.fromEvent(it)
+                    ?: IncomingRoomKeyRequest(localCreationTimestamp = 0L)
+        }
+    }
+
+    /**
+     * If you registered a `GossipingRequestListener`, you will be notified of key request
+     * that was not accepted by the SDK. You can call back this manually to accept anyhow.
+     */
+    override suspend fun manuallyAcceptRoomKeyRequest(request: IncomingRoomKeyRequest) {
+        incomingKeyRequestManager.manuallyAcceptRoomKeyRequest(request)
+    }
+
+    override fun getGossipingEventsTrail(): LiveData<PagedList<AuditTrail>> {
         return cryptoStore.getGossipingEventsTrail()
     }
 
-    override fun getGossipingEvents(): List<Event> {
+    override fun getGossipingEvents(): List<AuditTrail> {
         return cryptoStore.getGossipingEvents()
     }
 
@@ -1346,8 +1300,8 @@ internal class DefaultCryptoService @Inject constructor(
                 loadRoomMembersTask.execute(LoadRoomMembersTask.Params(roomId))
             } catch (failure: Throwable) {
                 Timber.tag(loggerTag.value).e("prepareToEncrypt() : Failed to load room members")
-                callback.onFailure(failure)
+                // we probably shouldn't block sending on that (but questionable)
-                return@launch
+                // but some members won't be able to decrypt
             }
 
             val userIds = getRoomUserIds(roomId)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/DeviceListManager.kt

@@ -315,10 +315,19 @@ internal class DeviceListManager @Inject constructor(
         } else {
             Timber.v("## CRYPTO | downloadKeys() : starts")
             val t0 = clock.epochMillis()
-            val result = doKeyDownloadForUsers(downloadUsers)
+            try {
-            Timber.v("## CRYPTO | downloadKeys() : doKeyDownloadForUsers succeeds after ${clock.epochMillis() - t0} ms")
+                val result = doKeyDownloadForUsers(downloadUsers)
-            result.also {
+                Timber.v("## CRYPTO | downloadKeys() : doKeyDownloadForUsers succeeds after ${clock.epochMillis() - t0} ms")
-                it.addEntriesFromMap(stored)
+                result.also {
+                    it.addEntriesFromMap(stored)
+                }
+            } catch (failure: Throwable) {
+                Timber.w(failure, "## CRYPTO | downloadKeys() : doKeyDownloadForUsers failed after ${clock.epochMillis() - t0} ms")
+                if (forceDownload) {
+                    throw failure
+                } else {
+                    stored
+                }
             }
         }
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/GossipingWorkManager.kt

@@ -1,58 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import androidx.work.BackoffPolicy
-import androidx.work.Data
-import androidx.work.ExistingWorkPolicy
-import androidx.work.ListenableWorker
-import androidx.work.OneTimeWorkRequest
-import org.matrix.android.sdk.api.util.Cancelable
-import org.matrix.android.sdk.internal.di.WorkManagerProvider
-import org.matrix.android.sdk.internal.session.SessionScope
-import org.matrix.android.sdk.internal.util.CancelableWork
-import org.matrix.android.sdk.internal.worker.startChain
-import java.util.UUID
-import java.util.concurrent.TimeUnit
-import javax.inject.Inject
-
-@SessionScope
-internal class GossipingWorkManager @Inject constructor(
-        private val workManagerProvider: WorkManagerProvider
-) {
-
-    inline fun <reified W : ListenableWorker> createWork(data: Data, startChain: Boolean): OneTimeWorkRequest {
-        return workManagerProvider.matrixOneTimeWorkRequestBuilder<W>()
-                .setConstraints(WorkManagerProvider.workConstraints)
-                .startChain(startChain)
-                .setInputData(data)
-                .setBackoffCriteria(BackoffPolicy.LINEAR, WorkManagerProvider.BACKOFF_DELAY_MILLIS, TimeUnit.MILLISECONDS)
-                .build()
-    }
-
-    // Prevent sending queue to stay broken after app restart
-    // The unique queue id will stay the same as long as this object is instantiated
-    private val queueSuffixApp = UUID.randomUUID()
-
-    fun postWork(workRequest: OneTimeWorkRequest, policy: ExistingWorkPolicy = ExistingWorkPolicy.APPEND): Cancelable {
-        workManagerProvider.workManager
-                .beginUniqueWork(this::class.java.name + "_$queueSuffixApp", policy, workRequest)
-                .enqueue()
-
-        return CancelableWork(workManagerProvider.workManager, workRequest.id)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/IncomingGossipingRequestManager.kt

@@ -1,475 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.launch
-import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
-import org.matrix.android.sdk.api.auth.data.Credentials
-import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_MEGOLM
-import org.matrix.android.sdk.api.crypto.MXCryptoConfig
-import org.matrix.android.sdk.api.session.crypto.crosssigning.KEYBACKUP_SECRET_SSSS_NAME
-import org.matrix.android.sdk.api.session.crypto.crosssigning.MASTER_KEY_SSSS_NAME
-import org.matrix.android.sdk.api.session.crypto.crosssigning.SELF_SIGNING_KEY_SSSS_NAME
-import org.matrix.android.sdk.api.session.crypto.crosssigning.USER_SIGNING_KEY_SSSS_NAME
-import org.matrix.android.sdk.api.session.crypto.keysbackup.extractCurveKeyFromRecoveryKey
-import org.matrix.android.sdk.api.session.crypto.keyshare.GossipingRequestListener
-import org.matrix.android.sdk.api.session.crypto.model.GossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.GossipingToDeviceObject
-import org.matrix.android.sdk.api.session.crypto.model.IncomingRequestCancellation
-import org.matrix.android.sdk.api.session.crypto.model.IncomingRoomKeyRequest
-import org.matrix.android.sdk.api.session.crypto.model.IncomingSecretShareRequest
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.api.util.toBase64NoPadding
-import org.matrix.android.sdk.internal.crypto.algorithms.IMXGroupEncryption
-import org.matrix.android.sdk.internal.crypto.model.rest.GossipingDefaultContent
-import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.createUniqueTxnId
-import org.matrix.android.sdk.internal.di.SessionId
-import org.matrix.android.sdk.internal.session.SessionScope
-import org.matrix.android.sdk.internal.util.time.Clock
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import timber.log.Timber
-import java.util.concurrent.Executors
-import javax.inject.Inject
-
-@SessionScope
-internal class IncomingGossipingRequestManager @Inject constructor(
-        @SessionId private val sessionId: String,
-        private val credentials: Credentials,
-        private val cryptoStore: IMXCryptoStore,
-        private val cryptoConfig: MXCryptoConfig,
-        private val gossipingWorkManager: GossipingWorkManager,
-        private val roomEncryptorsStore: RoomEncryptorsStore,
-        private val roomDecryptorProvider: RoomDecryptorProvider,
-        private val coroutineDispatchers: MatrixCoroutineDispatchers,
-        private val cryptoCoroutineScope: CoroutineScope,
-        private val clock: Clock,
-) {
-
-    private val executor = Executors.newSingleThreadExecutor()
-
-    // list of IncomingRoomKeyRequests/IncomingRoomKeyRequestCancellations
-    // we received in the current sync.
-    private val receivedGossipingRequests = ArrayList<IncomingShareRequestCommon>()
-    private val receivedRequestCancellations = ArrayList<IncomingRequestCancellation>()
-
-    // the listeners
-    private val gossipingRequestListeners: MutableSet<GossipingRequestListener> = HashSet()
-
-    init {
-        receivedGossipingRequests.addAll(cryptoStore.getPendingIncomingGossipingRequests())
-    }
-
-    fun close() {
-        executor.shutdownNow()
-    }
-
-    // Recently verified devices (map of deviceId and timestamp)
-    private val recentlyVerifiedDevices = HashMap<String, Long>()
-
-    /**
-     * Called when a session has been verified.
-     * This information can be used by the manager to decide whether or not to fullfil gossiping requests
-     */
-    fun onVerificationCompleteForDevice(deviceId: String) {
-        // For now we just keep an in memory cache
-        synchronized(recentlyVerifiedDevices) {
-            recentlyVerifiedDevices[deviceId] = clock.epochMillis()
-        }
-    }
-
-    private fun hasBeenVerifiedLessThanFiveMinutesFromNow(deviceId: String): Boolean {
-        val verifTimestamp: Long?
-        synchronized(recentlyVerifiedDevices) {
-            verifTimestamp = recentlyVerifiedDevices[deviceId]
-        }
-        if (verifTimestamp == null) return false
-
-        val age = clock.epochMillis() - verifTimestamp
-
-        return age < FIVE_MINUTES_IN_MILLIS
-    }
-
-    /**
-     * Called when we get an m.room_key_request event
-     * It must be called on CryptoThread
-     *
-     * @param event the announcement event.
-     */
-    fun onGossipingRequestEvent(event: Event) {
-        val roomKeyShare = event.getClearContent().toModel<GossipingDefaultContent>()
-        Timber.i("## CRYPTO | GOSSIP onGossipingRequestEvent received type ${event.type} from user:${event.senderId}, content:$roomKeyShare")
-        // val ageLocalTs = event.unsignedData?.age?.let { clock.epochMillis() - it }
-        when (roomKeyShare?.action) {
-            GossipingToDeviceObject.ACTION_SHARE_REQUEST      -> {
-                if (event.getClearType() == EventType.REQUEST_SECRET) {
-                    IncomingSecretShareRequest.fromEvent(event, clock.epochMillis())?.let {
-                        if (event.senderId == credentials.userId && it.deviceId == credentials.deviceId) {
-                            // ignore, it was sent by me as *
-                            Timber.v("## GOSSIP onGossipingRequestEvent type ${event.type} ignore remote echo")
-                        } else {
-//                            // save in DB
-//                            cryptoStore.storeIncomingGossipingRequest(it, ageLocalTs)
-                            receivedGossipingRequests.add(it)
-                        }
-                    }
-                } else if (event.getClearType() == EventType.ROOM_KEY_REQUEST) {
-                    IncomingRoomKeyRequest.fromEvent(event, clock.epochMillis())?.let {
-                        if (event.senderId == credentials.userId && it.deviceId == credentials.deviceId) {
-                            // ignore, it was sent by me as *
-                            Timber.v("## GOSSIP onGossipingRequestEvent type ${event.type} ignore remote echo")
-                        } else {
-//                            cryptoStore.storeIncomingGossipingRequest(it, ageLocalTs)
-                            receivedGossipingRequests.add(it)
-                        }
-                    }
-                }
-            }
-            GossipingToDeviceObject.ACTION_SHARE_CANCELLATION -> {
-                IncomingRequestCancellation.fromEvent(event, clock.epochMillis())?.let {
-                    receivedRequestCancellations.add(it)
-                }
-            }
-            else                                              -> {
-                Timber.e("## GOSSIP onGossipingRequestEvent() : unsupported action ${roomKeyShare?.action}")
-            }
-        }
-    }
-
-    /**
-     * Process any m.room_key_request or m.secret.request events which were queued up during the
-     * current sync.
-     * It must be called on CryptoThread
-     */
-    fun processReceivedGossipingRequests() {
-        val roomKeyRequestsToProcess = receivedGossipingRequests.toList()
-        receivedGossipingRequests.clear()
-
-        Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : ${roomKeyRequestsToProcess.size} request to process")
-
-        var receivedRequestCancellations: List<IncomingRequestCancellation>? = null
-
-        synchronized(this.receivedRequestCancellations) {
-            if (this.receivedRequestCancellations.isNotEmpty()) {
-                receivedRequestCancellations = this.receivedRequestCancellations.toList()
-                this.receivedRequestCancellations.clear()
-            }
-        }
-
-        executor.execute {
-            cryptoStore.storeIncomingGossipingRequests(roomKeyRequestsToProcess)
-            for (request in roomKeyRequestsToProcess) {
-                if (request is IncomingRoomKeyRequest) {
-                    processIncomingRoomKeyRequest(request)
-                } else if (request is IncomingSecretShareRequest) {
-                    processIncomingSecretShareRequest(request)
-                }
-            }
-
-            receivedRequestCancellations?.forEach { request ->
-                Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : m.room_key_request cancellation $request")
-                // we should probably only notify the app of cancellations we told it
-                // about, but we don't currently have a record of that, so we just pass
-                // everything through.
-                if (request.userId == credentials.userId && request.deviceId == credentials.deviceId) {
-                    // ignore remote echo
-                    return@forEach
-                }
-                val matchingIncoming = cryptoStore.getIncomingRoomKeyRequest(request.userId ?: "", request.deviceId ?: "", request.requestId ?: "")
-                if (matchingIncoming == null) {
-                    // ignore that?
-                    return@forEach
-                } else {
-                    // If it was accepted from this device, keep the information, do not mark as cancelled
-                    if (matchingIncoming.state != GossipingRequestState.ACCEPTED) {
-                        onRoomKeyRequestCancellation(request)
-                        cryptoStore.updateGossipingRequestState(request, GossipingRequestState.CANCELLED_BY_REQUESTER)
-                    }
-                }
-            }
-        }
-    }
-
-    private fun processIncomingRoomKeyRequest(request: IncomingRoomKeyRequest) {
-        val userId = request.userId ?: return
-        val deviceId = request.deviceId ?: return
-        val body = request.requestBody ?: return
-        val roomId = body.roomId ?: return
-        val alg = body.algorithm ?: return
-
-        Timber.v("## CRYPTO | GOSSIP processIncomingRoomKeyRequest from $userId:$deviceId for $roomId / ${body.sessionId} id ${request.requestId}")
-        if (credentials.userId != userId) {
-            handleKeyRequestFromOtherUser(body, request, alg, roomId, userId, deviceId)
-            return
-        }
-        // TODO: should we queue up requests we don't yet have keys for, in case they turn up later?
-        // if we don't have a decryptor for this room/alg, we don't have
-        // the keys for the requested events, and can drop the requests.
-        val decryptor = roomDecryptorProvider.getRoomDecryptor(roomId, alg)
-        if (null == decryptor) {
-            Timber.w("## CRYPTO | GOSSIP processReceivedGossipingRequests() : room key request for unknown $alg in room $roomId")
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            return
-        }
-        if (!decryptor.hasKeysForKeyRequest(request)) {
-            Timber.w("## CRYPTO | GOSSIP processReceivedGossipingRequests() : room key request for unknown session ${body.sessionId!!}")
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            return
-        }
-
-        if (credentials.deviceId == deviceId && credentials.userId == userId) {
-            Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : oneself device - ignored")
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            return
-        }
-        request.share = Runnable {
-            decryptor.shareKeysWithDevice(request)
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.ACCEPTED)
-        }
-        request.ignore = Runnable {
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-        }
-        // if the device is verified already, share the keys
-        val device = cryptoStore.getUserDevice(userId, deviceId)
-        if (device != null) {
-            if (device.isVerified) {
-                Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : device is already verified: sharing keys")
-                request.share?.run()
-                return
-            }
-
-            if (device.isBlocked) {
-                Timber.v("## CRYPTO | GOSSIP processReceivedGossipingRequests() : device is blocked -> ignored")
-                cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-                return
-            }
-        }
-
-        // As per config we automatically discard untrusted devices request
-        if (cryptoConfig.discardRoomKeyRequestsFromUntrustedDevices) {
-            Timber.v("## CRYPTO | processReceivedGossipingRequests() : discardRoomKeyRequestsFromUntrustedDevices")
-            // At this point the device is unknown, we don't want to bother user with that
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            return
-        }
-
-        // Pass to application layer to decide what to do
-        onRoomKeyRequest(request)
-    }
-
-    private fun handleKeyRequestFromOtherUser(body: RoomKeyRequestBody,
-                                              request: IncomingRoomKeyRequest,
-                                              alg: String,
-                                              roomId: String,
-                                              userId: String,
-                                              deviceId: String) {
-        Timber.w("## CRYPTO | GOSSIP processReceivedGossipingRequests() : room key request from other user")
-        val senderKey = body.senderKey ?: return Unit
-                .also { Timber.w("missing senderKey") }
-                .also { cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED) }
-        val sessionId = body.sessionId ?: return Unit
-                .also { Timber.w("missing sessionId") }
-                .also { cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED) }
-
-        if (alg != MXCRYPTO_ALGORITHM_MEGOLM) {
-            return Unit
-                    .also { Timber.w("Only megolm is accepted here") }
-                    .also { cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED) }
-        }
-
-        val roomEncryptor = roomEncryptorsStore.get(roomId) ?: return Unit
-                .also { Timber.w("no room Encryptor") }
-                .also { cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED) }
-
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-            if (roomEncryptor is IMXGroupEncryption) {
-                val isSuccess = roomEncryptor.reshareKey(sessionId, userId, deviceId, senderKey)
-
-                if (isSuccess) {
-                    cryptoStore.updateGossipingRequestState(request, GossipingRequestState.ACCEPTED)
-                } else {
-                    cryptoStore.updateGossipingRequestState(request, GossipingRequestState.UNABLE_TO_PROCESS)
-                }
-            } else {
-                Timber.e("## CRYPTO | handleKeyRequestFromOtherUser() from:$userId: Unable to handle IMXGroupEncryption.reshareKey for $alg")
-            }
-        }
-        cryptoStore.updateGossipingRequestState(request, GossipingRequestState.RE_REQUESTED)
-    }
-
-    private fun processIncomingSecretShareRequest(request: IncomingSecretShareRequest) {
-        val secretName = request.secretName ?: return Unit.also {
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            Timber.v("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Missing secret name")
-        }
-
-        val userId = request.userId
-        if (userId == null || credentials.userId != userId) {
-            Timber.e("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Ignoring secret share request from other users")
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            return
-        }
-
-        val deviceId = request.deviceId
-                ?: return Unit.also {
-                    Timber.e("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Malformed request, no ")
-                    cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-                }
-
-        val device = cryptoStore.getUserDevice(userId, deviceId)
-                ?: return Unit.also {
-                    Timber.e("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Received secret share request from unknown device ${request.deviceId}")
-                    cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-                }
-
-        if (!device.isVerified || device.isBlocked) {
-            Timber.v("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Ignoring secret share request from untrusted/blocked session $device")
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            return
-        }
-
-        val isDeviceLocallyVerified = cryptoStore.getUserDevice(userId, deviceId)?.trustLevel?.isLocallyVerified()
-
-        when (secretName) {
-            MASTER_KEY_SSSS_NAME       -> cryptoStore.getCrossSigningPrivateKeys()?.master
-            SELF_SIGNING_KEY_SSSS_NAME -> cryptoStore.getCrossSigningPrivateKeys()?.selfSigned
-            USER_SIGNING_KEY_SSSS_NAME -> cryptoStore.getCrossSigningPrivateKeys()?.user
-            KEYBACKUP_SECRET_SSSS_NAME -> cryptoStore.getKeyBackupRecoveryKeyInfo()?.recoveryKey
-                    ?.let {
-                        extractCurveKeyFromRecoveryKey(it)?.toBase64NoPadding()
-                    }
-            else                       -> null
-        }?.let { secretValue ->
-            Timber.i("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Sharing secret $secretName with $device locally trusted")
-            if (isDeviceLocallyVerified == true && hasBeenVerifiedLessThanFiveMinutesFromNow(deviceId)) {
-                val params = SendGossipWorker.Params(
-                        sessionId = sessionId,
-                        secretValue = secretValue,
-                        requestUserId = request.userId,
-                        requestDeviceId = request.deviceId,
-                        requestId = request.requestId,
-                        txnId = createUniqueTxnId()
-                )
-
-                cryptoStore.updateGossipingRequestState(request, GossipingRequestState.ACCEPTING)
-                val workRequest = gossipingWorkManager.createWork<SendGossipWorker>(WorkerParamsFactory.toData(params), true)
-                gossipingWorkManager.postWork(workRequest)
-            } else {
-                Timber.v("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : Can't share secret $secretName with $device, verification too old")
-                cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-            }
-            return
-        }
-
-        Timber.v("## CRYPTO | GOSSIP processIncomingSecretShareRequest() : $secretName unknown at SDK level, asking to app layer")
-
-        request.ignore = Runnable {
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.REJECTED)
-        }
-
-        request.share = { secretValue ->
-            val params = SendGossipWorker.Params(
-                    sessionId = userId,
-                    secretValue = secretValue,
-                    requestUserId = request.userId,
-                    requestDeviceId = request.deviceId,
-                    requestId = request.requestId,
-                    txnId = createUniqueTxnId()
-            )
-
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.ACCEPTING)
-            val workRequest = gossipingWorkManager.createWork<SendGossipWorker>(WorkerParamsFactory.toData(params), true)
-            gossipingWorkManager.postWork(workRequest)
-            cryptoStore.updateGossipingRequestState(request, GossipingRequestState.ACCEPTED)
-        }
-
-        onShareRequest(request)
-    }
-
-    /**
-     * Dispatch onRoomKeyRequest
-     *
-     * @param request the request
-     */
-    private fun onRoomKeyRequest(request: IncomingRoomKeyRequest) {
-        synchronized(gossipingRequestListeners) {
-            for (listener in gossipingRequestListeners) {
-                try {
-                    listener.onRoomKeyRequest(request)
-                } catch (e: Exception) {
-                    Timber.e(e, "## CRYPTO | onRoomKeyRequest() failed")
-                }
-            }
-        }
-    }
-
-    /**
-     * Ask for a value to the listeners, and take the first one
-     */
-    private fun onShareRequest(request: IncomingSecretShareRequest) {
-        synchronized(gossipingRequestListeners) {
-            for (listener in gossipingRequestListeners) {
-                try {
-                    if (listener.onSecretShareRequest(request)) {
-                        return
-                    }
-                } catch (e: Exception) {
-                    Timber.e(e, "## CRYPTO | GOSSIP onRoomKeyRequest() failed")
-                }
-            }
-        }
-        // Not handled, ignore
-        request.ignore?.run()
-    }
-
-    /**
-     * A room key request cancellation has been received.
-     *
-     * @param request the cancellation request
-     */
-    private fun onRoomKeyRequestCancellation(request: IncomingRequestCancellation) {
-        synchronized(gossipingRequestListeners) {
-            for (listener in gossipingRequestListeners) {
-                try {
-                    listener.onRoomKeyRequestCancellation(request)
-                } catch (e: Exception) {
-                    Timber.e(e, "## CRYPTO | GOSSIP onRoomKeyRequestCancellation() failed")
-                }
-            }
-        }
-    }
-
-    fun addRoomKeysRequestListener(listener: GossipingRequestListener) {
-        synchronized(gossipingRequestListeners) {
-            gossipingRequestListeners.add(listener)
-        }
-    }
-
-    fun removeRoomKeysRequestListener(listener: GossipingRequestListener) {
-        synchronized(gossipingRequestListeners) {
-            gossipingRequestListeners.remove(listener)
-        }
-    }
-
-    companion object {
-        private const val FIVE_MINUTES_IN_MILLIS = 5 * 60 * 1000
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/IncomingKeyRequestManager.kt

@@ -0,0 +1,463 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
+import org.matrix.android.sdk.api.auth.data.Credentials
+import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_MEGOLM
+import org.matrix.android.sdk.api.crypto.MXCryptoConfig
+import org.matrix.android.sdk.api.extensions.tryOrNull
+import org.matrix.android.sdk.api.logger.LoggerTag
+import org.matrix.android.sdk.api.session.crypto.keyshare.GossipingRequestListener
+import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
+import org.matrix.android.sdk.api.session.crypto.model.IncomingRoomKeyRequest
+import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
+import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
+import org.matrix.android.sdk.api.session.crypto.model.RoomKeyShareRequest
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.content.RoomKeyWithHeldContent
+import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
+import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
+import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
+import org.matrix.android.sdk.internal.session.SessionScope
+import org.matrix.android.sdk.internal.task.SemaphoreCoroutineSequencer
+import org.matrix.android.sdk.internal.util.time.Clock
+import timber.log.Timber
+import java.util.concurrent.Executors
+import javax.inject.Inject
+import kotlin.system.measureTimeMillis
+
+private val loggerTag = LoggerTag("IncomingKeyRequestManager", LoggerTag.CRYPTO)
+
+@SessionScope
+internal class IncomingKeyRequestManager @Inject constructor(
+        private val credentials: Credentials,
+        private val cryptoStore: IMXCryptoStore,
+        private val ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction,
+        private val olmDevice: MXOlmDevice,
+        private val cryptoConfig: MXCryptoConfig,
+        private val messageEncrypter: MessageEncrypter,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val sendToDeviceTask: SendToDeviceTask,
+        private val clock: Clock,
+) {
+
+    private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
+    private val outgoingRequestScope = CoroutineScope(SupervisorJob() + dispatcher)
+    val sequencer = SemaphoreCoroutineSequencer()
+
+    private val incomingRequestBuffer = mutableListOf<ValidMegolmRequestBody>()
+
+    // the listeners
+    private val gossipingRequestListeners: MutableSet<GossipingRequestListener> = HashSet()
+
+    enum class MegolmRequestAction {
+        Request, Cancel
+    }
+
+    data class ValidMegolmRequestBody(
+            val requestId: String,
+            val requestingUserId: String,
+            val requestingDeviceId: String,
+            val roomId: String,
+            val senderKey: String,
+            val sessionId: String,
+            val action: MegolmRequestAction
+    ) {
+        fun shortDbgString() = "Request from $requestingUserId|$requestingDeviceId for session $sessionId in room $roomId"
+    }
+
+    private fun RoomKeyShareRequest.toValidMegolmRequest(senderId: String): ValidMegolmRequestBody? {
+        val deviceId = requestingDeviceId ?: return null
+        val body = body ?: return null
+        val roomId = body.roomId ?: return null
+        val sessionId = body.sessionId ?: return null
+        val senderKey = body.senderKey ?: return null
+        val requestId = this.requestId ?: return null
+        if (body.algorithm != MXCRYPTO_ALGORITHM_MEGOLM) return null
+        val action = when (this.action) {
+            "request" -> MegolmRequestAction.Request
+            "request_cancellation" -> MegolmRequestAction.Cancel
+            else -> null
+        } ?: return null
+        return ValidMegolmRequestBody(
+                requestId = requestId,
+                requestingUserId = senderId,
+                requestingDeviceId = deviceId,
+                roomId = roomId,
+                senderKey = senderKey,
+                sessionId = sessionId,
+                action = action
+        )
+    }
+
+    fun addNewIncomingRequest(senderId: String, request: RoomKeyShareRequest) {
+        if (!cryptoStore.isKeyGossipingEnabled()) {
+            Timber.tag(loggerTag.value)
+                    .i("Ignore incoming key request as per crypto config in room ${request.body?.roomId}")
+            return
+        }
+        outgoingRequestScope.launch {
+            // It is important to handle requests in order
+            sequencer.post {
+                val validMegolmRequest = request.toValidMegolmRequest(senderId) ?: return@post Unit.also {
+                    Timber.tag(loggerTag.value).w("Received key request for unknown algorithm ${request.body?.algorithm}")
+                }
+
+                // is there already one like that?
+                val existing = incomingRequestBuffer.firstOrNull { it == validMegolmRequest }
+                if (existing == null) {
+                    when (validMegolmRequest.action) {
+                        MegolmRequestAction.Request -> {
+                            // just add to the buffer
+                            incomingRequestBuffer.add(validMegolmRequest)
+                        }
+                        MegolmRequestAction.Cancel  -> {
+                            // ignore, we can't cancel as it's not known (probably already processed)
+                            // still notify app layer if it was passed up previously
+                            IncomingRoomKeyRequest.fromRestRequest(senderId, request, clock)?.let { iReq ->
+                                outgoingRequestScope.launch(coroutineDispatchers.computation) {
+                                    val listenersCopy = synchronized(gossipingRequestListeners) {
+                                        gossipingRequestListeners.toList()
+                                    }
+                                    listenersCopy.onEach {
+                                        tryOrNull {
+                                            withContext(coroutineDispatchers.main) {
+                                                it.onRequestCancelled(iReq)
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    when (validMegolmRequest.action) {
+                        MegolmRequestAction.Request -> {
+                            // it's already in buffer, nop keep existing
+                        }
+                        MegolmRequestAction.Cancel  -> {
+                            // discard the request in buffer
+                            incomingRequestBuffer.remove(existing)
+                            outgoingRequestScope.launch(coroutineDispatchers.computation) {
+                                val listenersCopy = synchronized(gossipingRequestListeners) {
+                                    gossipingRequestListeners.toList()
+                                }
+                                listenersCopy.onEach {
+                                    IncomingRoomKeyRequest.fromRestRequest(senderId, request, clock)?.let { iReq ->
+                                        withContext(coroutineDispatchers.main) {
+                                            tryOrNull { it.onRequestCancelled(iReq) }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    fun processIncomingRequests() {
+        outgoingRequestScope.launch {
+            sequencer.post {
+                measureTimeMillis {
+                    Timber.tag(loggerTag.value).v("processIncomingKeyRequests : ${incomingRequestBuffer.size} request to process")
+                    incomingRequestBuffer.forEach {
+                        // should not happen, we only store requests
+                        if (it.action != MegolmRequestAction.Request) return@forEach
+                        try {
+                            handleIncomingRequest(it)
+                        } catch (failure: Throwable) {
+                            // ignore and continue, should not happen
+                            Timber.tag(loggerTag.value).w(failure, "processIncomingKeyRequests : failed to process request $it")
+                        }
+                    }
+                    incomingRequestBuffer.clear()
+                }.let { duration ->
+                    Timber.tag(loggerTag.value).v("Finish processing incoming key request in $duration ms")
+                }
+            }
+        }
+    }
+
+    private suspend fun handleIncomingRequest(request: ValidMegolmRequestBody) {
+        // We don't want to download keys, if we don't know the device yet we won't share any how?
+        val requestingDevice =
+                cryptoStore.getUserDevice(request.requestingUserId, request.requestingDeviceId)
+                        ?: return Unit.also {
+                            Timber.tag(loggerTag.value).d("Ignoring key request: ${request.shortDbgString()}")
+                        }
+
+        cryptoStore.saveIncomingKeyRequestAuditTrail(
+                request.requestId,
+                request.roomId,
+                request.sessionId,
+                request.senderKey,
+                MXCRYPTO_ALGORITHM_MEGOLM,
+                request.requestingUserId,
+                request.requestingDeviceId
+        )
+
+        val roomAlgorithm = // withContext(coroutineDispatchers.crypto) {
+                cryptoStore.getRoomAlgorithm(request.roomId)
+//        }
+        if (roomAlgorithm != MXCRYPTO_ALGORITHM_MEGOLM) {
+            // strange we received a request for a room that is not encrypted
+            // maybe a broken state?
+            Timber.tag(loggerTag.value).w("Received a key request in a room with unsupported alg:$roomAlgorithm , req:${request.shortDbgString()}")
+            return
+        }
+
+        // Is it for one of our sessions?
+        if (request.requestingUserId == credentials.userId) {
+            Timber.tag(loggerTag.value).v("handling request from own user: megolm session ${request.sessionId}")
+
+            if (request.requestingDeviceId == credentials.deviceId) {
+                // ignore it's a remote echo
+                return
+            }
+            // If it's verified we share from the early index we know
+            // if not we check if it was originaly shared or not
+            if (requestingDevice.isVerified) {
+                // we share from the earliest known chain index
+                shareMegolmKey(request, requestingDevice, null)
+            } else {
+                shareIfItWasPreviouslyShared(request, requestingDevice)
+            }
+        } else {
+            if (cryptoConfig.limitRoomKeyRequestsToMyDevices) {
+                Timber.tag(loggerTag.value).v("Ignore request from other user as per crypto config: ${request.shortDbgString()}")
+                return
+            }
+            Timber.tag(loggerTag.value).v("handling request from other user: megolm session ${request.sessionId}")
+            if (requestingDevice.isBlocked) {
+                // it's blocked, so send a withheld code
+                sendWithheldForRequest(request, WithHeldCode.BLACKLISTED)
+            } else {
+                shareIfItWasPreviouslyShared(request, requestingDevice)
+            }
+        }
+    }
+
+    private suspend fun shareIfItWasPreviouslyShared(request: ValidMegolmRequestBody, requestingDevice: CryptoDeviceInfo) {
+        // we don't reshare unless it was previously shared with
+        val wasSessionSharedWithUser = withContext(coroutineDispatchers.crypto) {
+            cryptoStore.getSharedSessionInfo(request.roomId, request.sessionId, requestingDevice)
+        }
+        if (wasSessionSharedWithUser.found && wasSessionSharedWithUser.chainIndex != null) {
+            // we share from the index it was previously shared with
+            shareMegolmKey(request, requestingDevice, wasSessionSharedWithUser.chainIndex.toLong())
+        } else {
+            val isOwnDevice = requestingDevice.userId == credentials.userId
+            sendWithheldForRequest(request, if (isOwnDevice) WithHeldCode.UNVERIFIED else WithHeldCode.UNAUTHORISED)
+            // if it's our device we could delegate to the app layer to decide
+            if (isOwnDevice) {
+                outgoingRequestScope.launch(coroutineDispatchers.computation) {
+                    val listenersCopy = synchronized(gossipingRequestListeners) {
+                        gossipingRequestListeners.toList()
+                    }
+                    val iReq = IncomingRoomKeyRequest(
+                            userId = requestingDevice.userId,
+                            deviceId = requestingDevice.deviceId,
+                            requestId = request.requestId,
+                            requestBody = RoomKeyRequestBody(
+                                    algorithm = MXCRYPTO_ALGORITHM_MEGOLM,
+                                    senderKey = request.senderKey,
+                                    sessionId = request.sessionId,
+                                    roomId = request.roomId
+                            ),
+                            localCreationTimestamp = clock.epochMillis()
+                    )
+                    listenersCopy.onEach {
+                        withContext(coroutineDispatchers.main) {
+                            tryOrNull { it.onRoomKeyRequest(iReq) }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private suspend fun sendWithheldForRequest(request: ValidMegolmRequestBody, code: WithHeldCode) {
+        Timber.tag(loggerTag.value)
+                .w("Send withheld $code for req: ${request.shortDbgString()}")
+        val withHeldContent = RoomKeyWithHeldContent(
+                roomId = request.roomId,
+                senderKey = request.senderKey,
+                algorithm = MXCRYPTO_ALGORITHM_MEGOLM,
+                sessionId = request.sessionId,
+                codeString = code.value,
+                fromDevice = credentials.deviceId
+        )
+
+        val params = SendToDeviceTask.Params(
+                EventType.ROOM_KEY_WITHHELD,
+                MXUsersDevicesMap<Any>().apply {
+                    setObject(request.requestingUserId, request.requestingDeviceId, withHeldContent)
+                }
+        )
+        try {
+            withContext(coroutineDispatchers.io) {
+                sendToDeviceTask.execute(params)
+                Timber.tag(loggerTag.value)
+                        .d("Send withheld $code req: ${request.shortDbgString()}")
+            }
+
+            cryptoStore.saveWithheldAuditTrail(
+                    roomId = request.roomId,
+                    sessionId = request.sessionId,
+                    senderKey = request.senderKey,
+                    algorithm = MXCRYPTO_ALGORITHM_MEGOLM,
+                    code = code,
+                    userId = request.requestingUserId,
+                    deviceId = request.requestingDeviceId
+            )
+        } catch (failure: Throwable) {
+            // Ignore it's not that important?
+            // do we want to fallback to a worker?
+            Timber.tag(loggerTag.value)
+                    .w("Failed to send withheld $code req: ${request.shortDbgString()} reason:${failure.localizedMessage}")
+        }
+    }
+
+    suspend fun manuallyAcceptRoomKeyRequest(request: IncomingRoomKeyRequest) {
+        request.requestId ?: return
+        request.deviceId ?: return
+        request.userId ?: return
+        request.requestBody?.roomId ?: return
+        request.requestBody.senderKey ?: return
+        request.requestBody.sessionId ?: return
+        val validReq = ValidMegolmRequestBody(
+                requestId = request.requestId,
+                requestingDeviceId = request.deviceId,
+                requestingUserId = request.userId,
+                roomId = request.requestBody.roomId,
+                senderKey = request.requestBody.senderKey,
+                sessionId = request.requestBody.sessionId,
+                action = MegolmRequestAction.Request
+        )
+        val requestingDevice =
+                cryptoStore.getUserDevice(request.userId, request.deviceId)
+                        ?: return Unit.also {
+                            Timber.tag(loggerTag.value).d("Ignoring key request: ${validReq.shortDbgString()}")
+                        }
+
+        shareMegolmKey(validReq, requestingDevice, null)
+    }
+
+    private suspend fun shareMegolmKey(validRequest: ValidMegolmRequestBody,
+                                       requestingDevice: CryptoDeviceInfo,
+                                       chainIndex: Long?): Boolean {
+        Timber.tag(loggerTag.value)
+                .d("try to re-share Megolm Key at index $chainIndex for ${validRequest.shortDbgString()}")
+
+        val devicesByUser = mapOf(validRequest.requestingUserId to listOf(requestingDevice))
+        val usersDeviceMap = try {
+            ensureOlmSessionsForDevicesAction.handle(devicesByUser)
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value)
+                    .w("Failed to establish olm session")
+            sendWithheldForRequest(validRequest, WithHeldCode.NO_OLM)
+            return false
+        }
+
+        val olmSessionResult = usersDeviceMap.getObject(requestingDevice.userId, requestingDevice.deviceId)
+        if (olmSessionResult?.sessionId == null) {
+            Timber.tag(loggerTag.value)
+                    .w("reshareKey: no session with this device, probably because there were no one-time keys")
+            sendWithheldForRequest(validRequest, WithHeldCode.NO_OLM)
+            return false
+        }
+        val sessionHolder = try {
+            olmDevice.getInboundGroupSession(validRequest.sessionId, validRequest.senderKey, validRequest.roomId)
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value)
+                    .e(failure, "shareKeysWithDevice: failed to get session ${validRequest.requestingUserId}")
+            // It's unavailable
+            sendWithheldForRequest(validRequest, WithHeldCode.UNAVAILABLE)
+            return false
+        }
+
+        val export = sessionHolder.mutex.withLock {
+            sessionHolder.wrapper.exportKeys(chainIndex)
+        } ?: return false.also {
+            Timber.tag(loggerTag.value)
+                    .e("shareKeysWithDevice: failed to export group session ${validRequest.sessionId}")
+        }
+
+        val payloadJson = mapOf(
+                "type" to EventType.FORWARDED_ROOM_KEY,
+                "content" to export
+        )
+
+        val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(requestingDevice))
+        val sendToDeviceMap = MXUsersDevicesMap<Any>()
+        sendToDeviceMap.setObject(requestingDevice.userId, requestingDevice.deviceId, encodedPayload)
+        Timber.tag(loggerTag.value).d("reshareKey() : try sending session ${validRequest.sessionId} to ${requestingDevice.shortDebugString()}")
+        val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
+        return try {
+            sendToDeviceTask.execute(sendToDeviceParams)
+            Timber.tag(loggerTag.value)
+                    .i("successfully re-shared session ${validRequest.sessionId} to ${requestingDevice.shortDebugString()}")
+            cryptoStore.saveForwardKeyAuditTrail(
+                    validRequest.roomId,
+                    validRequest.sessionId,
+                    validRequest.senderKey,
+                    MXCRYPTO_ALGORITHM_MEGOLM,
+                    requestingDevice.userId,
+                    requestingDevice.deviceId,
+                    chainIndex
+            )
+            true
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value)
+                    .e(failure, "fail to re-share session ${validRequest.sessionId} to ${requestingDevice.shortDebugString()}")
+            false
+        }
+    }
+
+    fun addRoomKeysRequestListener(listener: GossipingRequestListener) {
+        synchronized(gossipingRequestListeners) {
+            gossipingRequestListeners.add(listener)
+        }
+    }
+
+    fun removeRoomKeysRequestListener(listener: GossipingRequestListener) {
+        synchronized(gossipingRequestListeners) {
+            gossipingRequestListeners.remove(listener)
+        }
+    }
+
+    fun close() {
+        try {
+            outgoingRequestScope.cancel("User Terminate")
+            incomingRequestBuffer.clear()
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value).w("Failed to shutDown request manager")
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/IncomingShareRequestCommon.kt

@@ -1,36 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-internal interface IncomingShareRequestCommon {
-    /**
-     * The user id
-     */
-    val userId: String?
-
-    /**
-     * The device id
-     */
-    val deviceId: String?
-
-    /**
-     * The request id
-     */
-    val requestId: String?
-
-    val localCreationTimestamp: Long?
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/MXOlmDevice.kt

@@ -43,7 +43,6 @@ import org.matrix.olm.OlmOutboundGroupSession
 import org.matrix.olm.OlmSession
 import org.matrix.olm.OlmUtility
 import timber.log.Timber
-import java.net.URLEncoder
 import javax.inject.Inject
 
 private val loggerTag = LoggerTag("MXOlmDevice", LoggerTag.CRYPTO)
@@ -331,14 +330,6 @@ internal class MXOlmDevice @Inject constructor(
                 Timber.tag(loggerTag.value).e(e, "## createInboundSession() : removeOneTimeKeys failed")
             }
 
-            Timber.tag(loggerTag.value).v("## createInboundSession() : ciphertext: $ciphertext")
-            try {
-                val sha256 = olmUtility!!.sha256(URLEncoder.encode(ciphertext, "utf-8"))
-                Timber.tag(loggerTag.value).v("## createInboundSession() :ciphertext: SHA256: $sha256")
-            } catch (e: Exception) {
-                Timber.tag(loggerTag.value).e(e, "## createInboundSession() :ciphertext: cannot encode ciphertext")
-            }
-
             val olmMessage = OlmMessage()
             olmMessage.mCipherText = ciphertext
             olmMessage.mType = messageType.toLong()
@@ -589,6 +580,13 @@ internal class MXOlmDevice @Inject constructor(
 
     //  Inbound group session
 
+    sealed interface AddSessionResult {
+        data class Imported(val ratchetIndex: Int) : AddSessionResult
+        abstract class Failure : AddSessionResult
+        object NotImported : Failure()
+        data class NotImportedHigherIndex(val newIndex: Int) : Failure()
+    }
+
     /**
      * Add an inbound group session to the session store.
      *
@@ -607,7 +605,7 @@ internal class MXOlmDevice @Inject constructor(
                                senderKey: String,
                                forwardingCurve25519KeyChain: List<String>,
                                keysClaimed: Map<String, String>,
-                               exportFormat: Boolean): Boolean {
+                               exportFormat: Boolean): AddSessionResult {
         val candidateSession = OlmInboundGroupSessionWrapper2(sessionKey, exportFormat)
         val existingSessionHolder = tryOrNull { getInboundGroupSession(sessionId, senderKey, roomId) }
         val existingSession = existingSessionHolder?.wrapper
@@ -615,7 +613,7 @@ internal class MXOlmDevice @Inject constructor(
         if (existingSession != null) {
             Timber.tag(loggerTag.value).d("## addInboundGroupSession() check if known session is better than candidate session")
             try {
-                val existingFirstKnown = existingSession.firstKnownIndex ?: return false.also {
+                val existingFirstKnown = existingSession.firstKnownIndex ?: return AddSessionResult.NotImported.also {
                     // This is quite unexpected, could throw if native was released?
                     Timber.tag(loggerTag.value).e("## addInboundGroupSession() null firstKnownIndex on existing session")
                     candidateSession.olmInboundGroupSession?.releaseSession()
@@ -626,12 +624,12 @@ internal class MXOlmDevice @Inject constructor(
                 if (newKnownFirstIndex != null && existingFirstKnown <= newKnownFirstIndex) {
                     Timber.tag(loggerTag.value).d("## addInboundGroupSession() : ignore session our is better $senderKey/$sessionId")
                     candidateSession.olmInboundGroupSession?.releaseSession()
-                    return false
+                    return AddSessionResult.NotImportedHigherIndex(newKnownFirstIndex.toInt())
                 }
             } catch (failure: Throwable) {
                 Timber.tag(loggerTag.value).e("## addInboundGroupSession() Failed to add inbound: ${failure.localizedMessage}")
                 candidateSession.olmInboundGroupSession?.releaseSession()
-                return false
+                return AddSessionResult.NotImported
             }
         }
 
@@ -641,19 +639,19 @@ internal class MXOlmDevice @Inject constructor(
         val candidateOlmInboundSession = candidateSession.olmInboundGroupSession
         if (null == candidateOlmInboundSession) {
             Timber.tag(loggerTag.value).e("## addInboundGroupSession : invalid session <null>")
-            return false
+            return AddSessionResult.NotImported
         }
 
         try {
             if (candidateOlmInboundSession.sessionIdentifier() != sessionId) {
                 Timber.tag(loggerTag.value).e("## addInboundGroupSession : ERROR: Mismatched group session ID from senderKey: $senderKey")
                 candidateOlmInboundSession.releaseSession()
-                return false
+                return AddSessionResult.NotImported
             }
         } catch (e: Throwable) {
             candidateOlmInboundSession.releaseSession()
             Timber.tag(loggerTag.value).e(e, "## addInboundGroupSession : sessionIdentifier() failed")
-            return false
+            return AddSessionResult.NotImported
         }
 
         candidateSession.senderKey = senderKey
@@ -667,7 +665,7 @@ internal class MXOlmDevice @Inject constructor(
             inboundGroupSessionStore.storeInBoundGroupSession(InboundGroupSessionHolder(candidateSession), sessionId, senderKey)
         }
 
-        return true
+        return AddSessionResult.Imported(candidateSession.firstKnownIndex?.toInt() ?: 0)
     }
 
     /**
@@ -790,7 +788,7 @@ internal class MXOlmDevice @Inject constructor(
 
                 if (timelineSet.contains(messageIndexKey)) {
                     val reason = String.format(MXCryptoError.DUPLICATE_MESSAGE_INDEX_REASON, decryptResult.mIndex)
-                    Timber.tag(loggerTag.value).e("## decryptGroupMessage() : $reason")
+                    Timber.tag(loggerTag.value).e("## decryptGroupMessage() timelineId=$timeline: $reason")
                     throw MXCryptoError.Base(MXCryptoError.ErrorType.DUPLICATED_MESSAGE_INDEX, reason)
                 }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OutgoingGossipingRequest.kt

@@ -1,27 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-
-internal interface OutgoingGossipingRequest {
-    val recipients: Map<String, List<String>>
-    val requestId: String
-    val state: OutgoingGossipingRequestState
-    // transaction id for the cancellation, if any
-    // var cancellationTxnId: String?
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OutgoingGossipingRequestManager.kt

@@ -1,167 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.delay
-import kotlinx.coroutines.launch
-import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingRoomKeyRequest
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
-import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.createUniqueTxnId
-import org.matrix.android.sdk.internal.crypto.util.RequestIdHelper
-import org.matrix.android.sdk.internal.di.SessionId
-import org.matrix.android.sdk.internal.session.SessionScope
-import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
-import timber.log.Timber
-import javax.inject.Inject
-
-@SessionScope
-internal class OutgoingGossipingRequestManager @Inject constructor(
-        @SessionId private val sessionId: String,
-        private val cryptoStore: IMXCryptoStore,
-        private val coroutineDispatchers: MatrixCoroutineDispatchers,
-        private val cryptoCoroutineScope: CoroutineScope,
-        private val gossipingWorkManager: GossipingWorkManager) {
-
-    /**
-     * Send off a room key request, if we haven't already done so.
-     *
-     *
-     * The `requestBody` is compared (with a deep-equality check) against
-     * previous queued or sent requests and if it matches, no change is made.
-     * Otherwise, a request is added to the pending list, and a job is started
-     * in the background to send it.
-     *
-     * @param requestBody requestBody
-     * @param recipients  recipients
-     */
-    fun sendRoomKeyRequest(requestBody: RoomKeyRequestBody, recipients: Map<String, List<String>>) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-            cryptoStore.getOrAddOutgoingRoomKeyRequest(requestBody, recipients)?.let {
-                // Don't resend if it's already done, you need to cancel first (reRequest)
-                if (it.state == OutgoingGossipingRequestState.SENDING || it.state == OutgoingGossipingRequestState.SENT) {
-                    Timber.v("## CRYPTO - GOSSIP sendOutgoingRoomKeyRequest() : we already request for that session: $it")
-                    return@launch
-                }
-
-                sendOutgoingGossipingRequest(it)
-            }
-        }
-    }
-
-    fun sendSecretShareRequest(secretName: String, recipients: Map<String, List<String>>) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-            // A bit dirty, but for better stability give other party some time to mark
-            // devices trusted :/
-            delay(1500)
-            cryptoStore.getOrAddOutgoingSecretShareRequest(secretName, recipients)?.let {
-                // TODO check if there is already one that is being sent?
-                if (it.state == OutgoingGossipingRequestState.SENDING
-                /**|| it.state == OutgoingGossipingRequestState.SENT*/
-                ) {
-                    Timber.v("## CRYPTO - GOSSIP sendSecretShareRequest() : we are already sending for that session: $it")
-                    return@launch
-                }
-
-                sendOutgoingGossipingRequest(it)
-            }
-        }
-    }
-
-    /**
-     * Cancel room key requests, if any match the given details
-     *
-     * @param requestBody requestBody
-     */
-    fun cancelRoomKeyRequest(requestBody: RoomKeyRequestBody) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.computation) {
-            cancelRoomKeyRequest(requestBody, false)
-        }
-    }
-
-    /**
-     * Cancel room key requests, if any match the given details, and resend
-     *
-     * @param requestBody requestBody
-     */
-    fun resendRoomKeyRequest(requestBody: RoomKeyRequestBody) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.computation) {
-            cancelRoomKeyRequest(requestBody, true)
-        }
-    }
-
-    /**
-     * Cancel room key requests, if any match the given details, and resend
-     *
-     * @param requestBody requestBody
-     * @param andResend   true to resend the key request
-     */
-    private fun cancelRoomKeyRequest(requestBody: RoomKeyRequestBody, andResend: Boolean) {
-        val req = cryptoStore.getOutgoingRoomKeyRequest(requestBody) // no request was made for this key
-                ?: return Unit.also {
-                    Timber.v("## CRYPTO - GOSSIP cancelRoomKeyRequest() Unknown request $requestBody")
-                }
-
-        sendOutgoingRoomKeyRequestCancellation(req, andResend)
-    }
-
-    /**
-     * Send the outgoing key request.
-     *
-     * @param request the request
-     */
-    private fun sendOutgoingGossipingRequest(request: OutgoingGossipingRequest) {
-        Timber.v("## CRYPTO - GOSSIP sendOutgoingGossipingRequest() : Requesting keys $request")
-
-        val params = SendGossipRequestWorker.Params(
-                sessionId = sessionId,
-                keyShareRequest = request as? OutgoingRoomKeyRequest,
-                secretShareRequest = request as? OutgoingSecretRequest,
-                txnId = createUniqueTxnId()
-        )
-        cryptoStore.updateOutgoingGossipingRequestState(request.requestId, OutgoingGossipingRequestState.SENDING)
-        val workRequest = gossipingWorkManager.createWork<SendGossipRequestWorker>(WorkerParamsFactory.toData(params), true)
-        gossipingWorkManager.postWork(workRequest)
-    }
-
-    /**
-     * Given a OutgoingRoomKeyRequest, cancel it and delete the request record
-     *
-     * @param request the request
-     */
-    private fun sendOutgoingRoomKeyRequestCancellation(request: OutgoingRoomKeyRequest, resend: Boolean = false) {
-        Timber.v("## CRYPTO - sendOutgoingRoomKeyRequestCancellation $request")
-        val params = CancelGossipRequestWorker.Params.fromRequest(sessionId, request)
-        cryptoStore.updateOutgoingGossipingRequestState(request.requestId, OutgoingGossipingRequestState.CANCELLING)
-
-        val workRequest = gossipingWorkManager.createWork<CancelGossipRequestWorker>(WorkerParamsFactory.toData(params), true)
-        gossipingWorkManager.postWork(workRequest)
-
-        if (resend) {
-            val reSendParams = SendGossipRequestWorker.Params(
-                    sessionId = sessionId,
-                    keyShareRequest = request.copy(requestId = RequestIdHelper.createUniqueRequestId()),
-                    txnId = createUniqueTxnId()
-            )
-            val reSendWorkRequest = gossipingWorkManager.createWork<SendGossipRequestWorker>(WorkerParamsFactory.toData(reSendParams), true)
-            gossipingWorkManager.postWork(reSendWorkRequest)
-        }
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OutgoingKeyRequestManager.kt

@@ -0,0 +1,518 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
+import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_MEGOLM
+import org.matrix.android.sdk.api.crypto.MXCryptoConfig
+import org.matrix.android.sdk.api.extensions.tryOrNull
+import org.matrix.android.sdk.api.logger.LoggerTag
+import org.matrix.android.sdk.api.session.crypto.OutgoingKeyRequest
+import org.matrix.android.sdk.api.session.crypto.OutgoingRoomKeyRequestState
+import org.matrix.android.sdk.api.session.crypto.model.GossipingToDeviceObject
+import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
+import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
+import org.matrix.android.sdk.api.session.crypto.model.RoomKeyShareRequest
+import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
+import org.matrix.android.sdk.api.session.events.model.content.RoomKeyWithHeldContent
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.util.fromBase64
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
+import org.matrix.android.sdk.internal.di.SessionId
+import org.matrix.android.sdk.internal.di.UserId
+import org.matrix.android.sdk.internal.session.SessionScope
+import org.matrix.android.sdk.internal.task.SemaphoreCoroutineSequencer
+import timber.log.Timber
+import java.util.Stack
+import java.util.concurrent.Executors
+import javax.inject.Inject
+import kotlin.system.measureTimeMillis
+
+private val loggerTag = LoggerTag("OutgoingKeyRequestManager", LoggerTag.CRYPTO)
+
+/**
+ * This class is responsible for sending key requests to other devices when a message failed to decrypt.
+ * It's lifecycle is based on the sync pulse:
+ *    - You can post queries for session, or report when you got a session
+ *    - At the end of the sync (onSyncComplete) it will then process all the posted request and send to devices
+ * If a request failed it will be retried at the end of the next sync
+ */
+@SessionScope
+internal class OutgoingKeyRequestManager @Inject constructor(
+        @SessionId private val sessionId: String,
+        @UserId private val myUserId: String,
+        private val cryptoStore: IMXCryptoStore,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val cryptoConfig: MXCryptoConfig,
+        private val inboundGroupSessionStore: InboundGroupSessionStore,
+        private val sendToDeviceTask: SendToDeviceTask,
+        private val deviceListManager: DeviceListManager,
+        private val perSessionBackupQueryRateLimiter: PerSessionBackupQueryRateLimiter) {
+
+    private val dispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
+    private val outgoingRequestScope = CoroutineScope(SupervisorJob() + dispatcher)
+    private val sequencer = SemaphoreCoroutineSequencer()
+
+    // We only have one active key request per session, so we don't request if it's already requested
+    // But it could make sense to check more the backup, as it's evolving.
+    // We keep a stack as we consider that the key requested last is more likely to be on screen?
+    private val requestDiscardedBecauseAlreadySentThatCouldBeTriedWithBackup = Stack<Pair<String, String>>()
+
+    fun requestKeyForEvent(event: Event, force: Boolean) {
+        val (targets, body) = getRoomKeyRequestTargetForEvent(event) ?: return
+        val index = ratchetIndexForMessage(event) ?: 0
+        postRoomKeyRequest(body, targets, index, force)
+    }
+
+    private fun getRoomKeyRequestTargetForEvent(event: Event): Pair<Map<String, List<String>>, RoomKeyRequestBody>? {
+        val sender = event.senderId ?: return null
+        val encryptedEventContent = event.content.toModel<EncryptedEventContent>() ?: return null.also {
+            Timber.tag(loggerTag.value).e("getRoomKeyRequestTargetForEvent Failed to re-request key, null content")
+        }
+        if (encryptedEventContent.algorithm != MXCRYPTO_ALGORITHM_MEGOLM) return null
+
+        val senderDevice = encryptedEventContent.deviceId
+        val recipients = if (cryptoConfig.limitRoomKeyRequestsToMyDevices) {
+            mapOf(
+                    myUserId to listOf("*")
+            )
+        } else {
+            if (event.senderId == myUserId) {
+                mapOf(
+                        myUserId to listOf("*")
+                )
+            } else {
+                // for the case where you share the key with a device that has a broken olm session
+                // The other user might Re-shares a megolm session key with devices if the key has already been
+                // sent to them.
+                mapOf(
+                        myUserId to listOf("*"),
+
+                        // We might not have deviceId in the future due to https://github.com/matrix-org/matrix-spec-proposals/pull/3700
+                        // so in this case query to all
+                        sender to listOf(senderDevice ?: "*")
+                )
+            }
+        }
+
+        val requestBody = RoomKeyRequestBody(
+                roomId = event.roomId,
+                algorithm = encryptedEventContent.algorithm,
+                senderKey = encryptedEventContent.senderKey,
+                sessionId = encryptedEventContent.sessionId
+        )
+        return recipients to requestBody
+    }
+
+    private fun ratchetIndexForMessage(event: Event): Int? {
+        val encryptedContent = event.content.toModel<EncryptedEventContent>() ?: return null
+        if (encryptedContent.algorithm != MXCRYPTO_ALGORITHM_MEGOLM) return null
+        return encryptedContent.ciphertext?.fromBase64()?.inputStream()?.reader()?.let {
+            tryOrNull {
+                val megolmVersion = it.read()
+                if (megolmVersion != 3) return@tryOrNull null
+                /** Int tag */
+                if (it.read() != 8) return@tryOrNull null
+                it.read()
+            }
+        }
+    }
+
+    fun postRoomKeyRequest(requestBody: RoomKeyRequestBody, recipients: Map<String, List<String>>, fromIndex: Int, force: Boolean = false) {
+        outgoingRequestScope.launch {
+            sequencer.post {
+                internalQueueRequest(requestBody, recipients, fromIndex, force)
+            }
+        }
+    }
+
+    /**
+     * Typically called when we the session as been imported or received meanwhile
+     */
+    fun postCancelRequestForSessionIfNeeded(sessionId: String, roomId: String, senderKey: String, fromIndex: Int) {
+        outgoingRequestScope.launch {
+            sequencer.post {
+                internalQueueCancelRequest(sessionId, roomId, senderKey, fromIndex)
+            }
+        }
+    }
+
+    fun onSelfCrossSigningTrustChanged(newTrust: Boolean) {
+        if (newTrust) {
+            // we were previously not cross signed, but we are now
+            // so there is now more chances to get better replies for existing request
+            // Let's forget about sent request so that next time we try to decrypt we will resend requests
+            // We don't resend all because we don't want to generate a bulk of traffic
+            outgoingRequestScope.launch {
+                sequencer.post {
+                    cryptoStore.deleteOutgoingRoomKeyRequestInState(OutgoingRoomKeyRequestState.SENT)
+                }
+
+                sequencer.post {
+                    delay(1000)
+                    perSessionBackupQueryRateLimiter.refreshBackupInfoIfNeeded(true)
+                }
+            }
+        }
+    }
+
+    fun onRoomKeyForwarded(sessionId: String,
+                           algorithm: String,
+                           roomId: String,
+                           senderKey: String,
+                           fromDevice: String?,
+                           fromIndex: Int,
+                           event: Event) {
+        Timber.tag(loggerTag.value).d("Key forwarded for $sessionId from ${event.senderId}|$fromDevice at index $fromIndex")
+        outgoingRequestScope.launch {
+            sequencer.post {
+                cryptoStore.updateOutgoingRoomKeyReply(
+                        roomId = roomId,
+                        sessionId = sessionId,
+                        algorithm = algorithm,
+                        senderKey = senderKey,
+                        fromDevice = fromDevice,
+                        // strip out encrypted stuff as it's just a trail?
+                        event = event.copy(
+                                type = event.getClearType(),
+                                content = mapOf(
+                                        "chain_index" to fromIndex
+                                )
+                        )
+                )
+            }
+        }
+    }
+
+    fun onRoomKeyWithHeld(sessionId: String,
+                          algorithm: String,
+                          roomId: String,
+                          senderKey: String,
+                          fromDevice: String?,
+                          event: Event) {
+        outgoingRequestScope.launch {
+            sequencer.post {
+                Timber.tag(loggerTag.value).d("Withheld received for $sessionId from ${event.senderId}|$fromDevice")
+                Timber.tag(loggerTag.value).v("Withheld content ${event.getClearContent()}")
+
+                // We want to store withheld code from the sender of the message (owner of the megolm session), not from
+                // other devices that might gossip the key. If not the initial reason might be overridden
+                // by a request to one of our session.
+                event.getClearContent().toModel<RoomKeyWithHeldContent>()?.let { withheld ->
+                    withContext(coroutineDispatchers.crypto) {
+                        tryOrNull {
+                            deviceListManager.downloadKeys(listOf(event.senderId ?: ""), false)
+                        }
+                        cryptoStore.getUserDeviceList(event.senderId ?: "")
+                                .also { devices ->
+                                    Timber.tag(loggerTag.value)
+                                            .v("Withheld Devices for ${event.senderId} are ${devices.orEmpty().joinToString { it.identityKey() ?: "" }}")
+                                }
+                                ?.firstOrNull {
+                                    it.identityKey() == senderKey
+                                }
+                    }.also {
+                        Timber.tag(loggerTag.value).v("Withheld device for sender key $senderKey is from ${it?.shortDebugString()}")
+                    }?.let {
+                        if (it.userId == event.senderId) {
+                            if (fromDevice != null) {
+                                if (it.deviceId == fromDevice) {
+                                    Timber.tag(loggerTag.value).v("Storing sender Withheld code ${withheld.code} for ${withheld.sessionId}")
+                                    cryptoStore.addWithHeldMegolmSession(withheld)
+                                }
+                            } else {
+                                Timber.tag(loggerTag.value).v("Storing sender Withheld code ${withheld.code} for ${withheld.sessionId}")
+                                cryptoStore.addWithHeldMegolmSession(withheld)
+                            }
+                        }
+                    }
+                }
+
+                // Here we store the replies from a given request
+                cryptoStore.updateOutgoingRoomKeyReply(
+                        roomId = roomId,
+                        sessionId = sessionId,
+                        algorithm = algorithm,
+                        senderKey = senderKey,
+                        fromDevice = fromDevice,
+                        event = event
+                )
+            }
+        }
+    }
+
+    /**
+     * Should be called after a sync, ideally if no catchup sync needed (as keys might arrive in those)
+     */
+    fun requireProcessAllPendingKeyRequests() {
+        outgoingRequestScope.launch {
+            sequencer.post {
+                internalProcessPendingKeyRequests()
+            }
+        }
+    }
+
+    private fun internalQueueCancelRequest(sessionId: String, roomId: String, senderKey: String, localKnownChainIndex: Int) {
+        // do we have known requests for that session??
+        Timber.tag(loggerTag.value).v("Cancel Key Request if needed for $sessionId")
+        val knownRequest = cryptoStore.getOutgoingRoomKeyRequest(
+                algorithm = MXCRYPTO_ALGORITHM_MEGOLM,
+                roomId = roomId,
+                sessionId = sessionId,
+                senderKey = senderKey
+        )
+        if (knownRequest.isEmpty()) return Unit.also {
+            Timber.tag(loggerTag.value).v("Handle Cancel Key Request for $sessionId -- Was not currently requested")
+        }
+        if (knownRequest.size > 1) {
+            // It's worth logging, there should be only one
+            Timber.tag(loggerTag.value).w("Found multiple requests for same sessionId $sessionId")
+        }
+        knownRequest.forEach { request ->
+            when (request.state) {
+                OutgoingRoomKeyRequestState.UNSENT                               -> {
+                    if (request.fromIndex >= localKnownChainIndex) {
+                        // we have a good index we can cancel
+                        cryptoStore.deleteOutgoingRoomKeyRequest(request.requestId)
+                    }
+                }
+                OutgoingRoomKeyRequestState.SENT                                 -> {
+                    // It was already sent, and index satisfied we can cancel
+                    if (request.fromIndex >= localKnownChainIndex) {
+                        cryptoStore.updateOutgoingRoomKeyRequestState(request.requestId, OutgoingRoomKeyRequestState.CANCELLATION_PENDING)
+                    }
+                }
+                OutgoingRoomKeyRequestState.CANCELLATION_PENDING                 -> {
+                    // It is already marked to be cancelled
+                }
+                OutgoingRoomKeyRequestState.CANCELLATION_PENDING_AND_WILL_RESEND -> {
+                    if (request.fromIndex >= localKnownChainIndex) {
+                        // we just want to cancel now
+                        cryptoStore.updateOutgoingRoomKeyRequestState(request.requestId, OutgoingRoomKeyRequestState.CANCELLATION_PENDING)
+                    }
+                }
+                OutgoingRoomKeyRequestState.SENT_THEN_CANCELED                   -> {
+                    // was already canceled
+                    // if we need a better index, should we resend?
+                }
+            }
+        }
+    }
+
+    fun close() {
+        try {
+            outgoingRequestScope.cancel("User Terminate")
+            requestDiscardedBecauseAlreadySentThatCouldBeTriedWithBackup.clear()
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value).w("Failed to shutDown request manager")
+        }
+    }
+
+    private fun internalQueueRequest(requestBody: RoomKeyRequestBody, recipients: Map<String, List<String>>, fromIndex: Int, force: Boolean) {
+        if (!cryptoStore.isKeyGossipingEnabled()) {
+            // we might want to try backup?
+            if (requestBody.roomId != null && requestBody.sessionId != null) {
+                requestDiscardedBecauseAlreadySentThatCouldBeTriedWithBackup.push(requestBody.roomId to requestBody.sessionId)
+            }
+            Timber.tag(loggerTag.value).d("discarding request for ${requestBody.sessionId} as gossiping is disabled")
+            return
+        }
+
+        Timber.tag(loggerTag.value).d("Queueing key request for ${requestBody.sessionId} force:$force")
+        val existing = cryptoStore.getOutgoingRoomKeyRequest(requestBody)
+        Timber.tag(loggerTag.value).v("Queueing key request exiting is ${existing?.state}")
+        when (existing?.state) {
+            null                                                             -> {
+                // create a new one
+                cryptoStore.getOrAddOutgoingRoomKeyRequest(requestBody, recipients, fromIndex)
+            }
+            OutgoingRoomKeyRequestState.UNSENT                               -> {
+                // nothing it's new or not yet handled
+            }
+            OutgoingRoomKeyRequestState.SENT                                 -> {
+                // it was already requested
+                Timber.tag(loggerTag.value).d("The session ${requestBody.sessionId} is already requested")
+                if (force) {
+                    // update to UNSENT
+                    Timber.tag(loggerTag.value).d(".. force to request  ${requestBody.sessionId}")
+                    cryptoStore.updateOutgoingRoomKeyRequestState(existing.requestId, OutgoingRoomKeyRequestState.CANCELLATION_PENDING_AND_WILL_RESEND)
+                } else {
+                    if (existing.roomId != null && existing.sessionId != null) {
+                        requestDiscardedBecauseAlreadySentThatCouldBeTriedWithBackup.push(existing.roomId to existing.sessionId)
+                    }
+                }
+            }
+            OutgoingRoomKeyRequestState.CANCELLATION_PENDING                 -> {
+                // request is canceled only if I got the keys so what to do here...
+                if (force) {
+                    cryptoStore.updateOutgoingRoomKeyRequestState(existing.requestId, OutgoingRoomKeyRequestState.CANCELLATION_PENDING_AND_WILL_RESEND)
+                }
+            }
+            OutgoingRoomKeyRequestState.CANCELLATION_PENDING_AND_WILL_RESEND -> {
+                // It's already going to resend
+            }
+            OutgoingRoomKeyRequestState.SENT_THEN_CANCELED                   -> {
+                if (force) {
+                    cryptoStore.deleteOutgoingRoomKeyRequest(existing.requestId)
+                    cryptoStore.getOrAddOutgoingRoomKeyRequest(requestBody, recipients, fromIndex)
+                }
+            }
+        }
+
+        if (existing != null && existing.fromIndex >= fromIndex) {
+            // update the required index
+            cryptoStore.updateOutgoingRoomKeyRequiredIndex(existing.requestId, fromIndex)
+        }
+    }
+
+    private suspend fun internalProcessPendingKeyRequests() {
+        val toProcess = cryptoStore.getOutgoingRoomKeyRequests(OutgoingRoomKeyRequestState.pendingStates())
+        Timber.tag(loggerTag.value).v("Processing all pending key requests (found ${toProcess.size} pending)")
+
+        measureTimeMillis {
+            toProcess.forEach {
+                when (it.state) {
+                    OutgoingRoomKeyRequestState.UNSENT                               -> handleUnsentRequest(it)
+                    OutgoingRoomKeyRequestState.CANCELLATION_PENDING                 -> handleRequestToCancel(it)
+                    OutgoingRoomKeyRequestState.CANCELLATION_PENDING_AND_WILL_RESEND -> handleRequestToCancelWillResend(it)
+                    OutgoingRoomKeyRequestState.SENT_THEN_CANCELED,
+                    OutgoingRoomKeyRequestState.SENT                                 -> {
+                        // these are filtered out
+                    }
+                }
+            }
+        }.let {
+            Timber.tag(loggerTag.value).v("Finish processing pending key request in $it ms")
+        }
+
+        val maxBackupCallsBySync = 60
+        var currentCalls = 0
+        measureTimeMillis {
+            while (requestDiscardedBecauseAlreadySentThatCouldBeTriedWithBackup.isNotEmpty() && currentCalls < maxBackupCallsBySync) {
+                requestDiscardedBecauseAlreadySentThatCouldBeTriedWithBackup.pop().let { (roomId, sessionId) ->
+                    // we want to rate limit that somehow :/
+                    perSessionBackupQueryRateLimiter.tryFromBackupIfPossible(sessionId, roomId)
+                }
+                currentCalls++
+            }
+        }.let {
+            Timber.tag(loggerTag.value).v("Finish querying backup in $it ms")
+        }
+    }
+
+    private suspend fun handleUnsentRequest(request: OutgoingKeyRequest) {
+        // In order to avoid generating to_device traffic, we can first check if the key is backed up
+        Timber.tag(loggerTag.value).v("Handling unsent request for megolm session ${request.sessionId} in ${request.roomId}")
+        val sessionId = request.sessionId ?: return
+        val roomId = request.roomId ?: return
+        if (perSessionBackupQueryRateLimiter.tryFromBackupIfPossible(sessionId, roomId)) {
+            // let's see what's the index
+            val knownIndex = tryOrNull {
+                inboundGroupSessionStore.getInboundGroupSession(sessionId, request.requestBody?.senderKey ?: "")?.wrapper?.firstKnownIndex
+            }
+            if (knownIndex != null && knownIndex <= request.fromIndex) {
+                // we found the key in backup with good enough index, so we can just mark as cancelled, no need to send request
+                Timber.tag(loggerTag.value).v("Megolm session $sessionId successfully restored from backup, do not send request")
+                cryptoStore.deleteOutgoingRoomKeyRequest(request.requestId)
+                return
+            }
+        }
+
+        // we need to send the request
+        val toDeviceContent = RoomKeyShareRequest(
+                requestingDeviceId = cryptoStore.getDeviceId(),
+                requestId = request.requestId,
+                action = GossipingToDeviceObject.ACTION_SHARE_REQUEST,
+                body = request.requestBody
+        )
+        val contentMap = MXUsersDevicesMap<Any>()
+        request.recipients.forEach { userToDeviceMap ->
+            userToDeviceMap.value.forEach { deviceId ->
+                contentMap.setObject(userToDeviceMap.key, deviceId, toDeviceContent)
+            }
+        }
+
+        val params = SendToDeviceTask.Params(
+                eventType = EventType.ROOM_KEY_REQUEST,
+                contentMap = contentMap,
+                transactionId = request.requestId
+        )
+        try {
+            withContext(coroutineDispatchers.io) {
+                sendToDeviceTask.executeRetry(params, 3)
+            }
+            Timber.tag(loggerTag.value).d("Key request sent for $sessionId in room $roomId to ${request.recipients}")
+            // The request was sent, so update state
+            cryptoStore.updateOutgoingRoomKeyRequestState(request.requestId, OutgoingRoomKeyRequestState.SENT)
+            // TODO update the audit trail
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value).v("Failed to request $sessionId targets:${request.recipients}")
+        }
+    }
+
+    private suspend fun handleRequestToCancel(request: OutgoingKeyRequest): Boolean {
+        Timber.tag(loggerTag.value).v("handleRequestToCancel for megolm session ${request.sessionId}")
+        // we have to cancel this
+        val toDeviceContent = RoomKeyShareRequest(
+                requestingDeviceId = cryptoStore.getDeviceId(),
+                requestId = request.requestId,
+                action = GossipingToDeviceObject.ACTION_SHARE_CANCELLATION
+        )
+        val contentMap = MXUsersDevicesMap<Any>()
+        request.recipients.forEach { userToDeviceMap ->
+            userToDeviceMap.value.forEach { deviceId ->
+                contentMap.setObject(userToDeviceMap.key, deviceId, toDeviceContent)
+            }
+        }
+
+        val params = SendToDeviceTask.Params(
+                eventType = EventType.ROOM_KEY_REQUEST,
+                contentMap = contentMap,
+                transactionId = request.requestId
+        )
+        return try {
+            withContext(coroutineDispatchers.io) {
+                sendToDeviceTask.executeRetry(params, 3)
+            }
+            // The request cancellation was sent, we don't delete yet because we want
+            // to keep trace of the sent replies
+            cryptoStore.updateOutgoingRoomKeyRequestState(request.requestId, OutgoingRoomKeyRequestState.SENT_THEN_CANCELED)
+            true
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value).v("Failed to cancel request ${request.requestId} for session $sessionId targets:${request.recipients}")
+            false
+        }
+    }
+
+    private suspend fun handleRequestToCancelWillResend(request: OutgoingKeyRequest) {
+        if (handleRequestToCancel(request)) {
+            // this will create a new unsent request with no replies that will be process in the following call
+            cryptoStore.deleteOutgoingRoomKeyRequest(request.requestId)
+            request.requestBody?.let { cryptoStore.getOrAddOutgoingRoomKeyRequest(it, request.recipients, request.fromIndex) }
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/OutgoingSecretRequest.kt

@@ -1,39 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-
-/**
- * Represents an outgoing room key request
- */
-@JsonClass(generateAdapter = true)
-internal class OutgoingSecretRequest(
-        // Secret Name
-        val secretName: String?,
-        // list of recipients for the request
-        override var recipients: Map<String, List<String>>,
-        // Unique id for this request. Used for both
-        // an id within the request for later pairing with a cancellation, and for
-        // the transaction id when sending the to_device messages to our local
-        override var requestId: String,
-        // current state of this request
-        override var state: OutgoingGossipingRequestState) : OutgoingGossipingRequest {
-
-    // transaction id for the cancellation, if any
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/PerSessionBackupQueryRateLimiter.kt

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import dagger.Lazy
+import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
+import org.matrix.android.sdk.api.logger.LoggerTag
+import org.matrix.android.sdk.api.session.crypto.keysbackup.KeysVersionResult
+import org.matrix.android.sdk.api.session.crypto.keysbackup.SavedKeyBackupKeyInfo
+import org.matrix.android.sdk.api.session.crypto.model.ImportRoomKeysResult
+import org.matrix.android.sdk.api.util.awaitCallback
+import org.matrix.android.sdk.internal.crypto.keysbackup.DefaultKeysBackupService
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.util.time.Clock
+import timber.log.Timber
+import javax.inject.Inject
+
+// I keep the same name as OutgoingGossipingRequestManager to ease filtering of logs
+private val loggerTag = LoggerTag("OutgoingGossipingRequestManager", LoggerTag.CRYPTO)
+
+/**
+ * Used to try to get the key for UISI messages before sending room key request.
+ * We are adding some rate limiting to avoid querying too much for a key not in backup.
+ * Nonetheless the backup can be updated so we might want to retry from time to time.
+ */
+internal class PerSessionBackupQueryRateLimiter @Inject constructor(
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val keysBackupService: Lazy<DefaultKeysBackupService>,
+        private val cryptoStore: IMXCryptoStore,
+        private val clock: Clock,
+) {
+
+    companion object {
+        const val MIN_TRY_BACKUP_PERIOD_MILLIS = 60 * 60_000 // 1 hour
+    }
+
+    data class Info(
+            val megolmSessionId: String,
+            val roomId: String
+    )
+
+    data class LastTry(
+            val backupVersion: String,
+            val timestamp: Long
+    )
+
+    /**
+     * Remember what we already tried (a key not in backup or some server issue)
+     * We might want to retry from time to time as the backup could have been updated
+     */
+    private val lastFailureMap = mutableMapOf<Info, LastTry>()
+
+    private var backupVersion: KeysVersionResult? = null
+    private var savedKeyBackupKeyInfo: SavedKeyBackupKeyInfo? = null
+    var backupWasCheckedFromServer: Boolean = false
+    val now = clock.epochMillis()
+
+    fun refreshBackupInfoIfNeeded(force: Boolean = false) {
+        if (backupWasCheckedFromServer && !force) return
+        Timber.tag(loggerTag.value).v("Checking if can access a backup")
+        backupWasCheckedFromServer = true
+        val knownBackupSecret = cryptoStore.getKeyBackupRecoveryKeyInfo()
+                ?: return Unit.also {
+                    Timber.tag(loggerTag.value).v("We don't have the backup secret!")
+                }
+        this.backupVersion = keysBackupService.get().keysBackupVersion
+        this.savedKeyBackupKeyInfo = knownBackupSecret
+    }
+
+    suspend fun tryFromBackupIfPossible(sessionId: String, roomId: String): Boolean {
+        Timber.tag(loggerTag.value).v("tryFromBackupIfPossible for session:$sessionId in $roomId")
+        refreshBackupInfoIfNeeded()
+        val currentVersion = backupVersion
+        if (savedKeyBackupKeyInfo?.version == null ||
+                currentVersion == null ||
+                currentVersion.version != savedKeyBackupKeyInfo?.version) {
+            // We can't access the backup
+            Timber.tag(loggerTag.value).v("Can't get backup version info")
+            return false
+        }
+        val cacheKey = Info(sessionId, roomId)
+        val lastTry = lastFailureMap[cacheKey]
+        val shouldQuery =
+                lastTry == null ||
+                        lastTry.backupVersion != currentVersion.version ||
+                        (now - lastTry.timestamp) > MIN_TRY_BACKUP_PERIOD_MILLIS
+
+        if (!shouldQuery) return false
+
+        val successfullyImported = withContext(coroutineDispatchers.io) {
+            try {
+                awaitCallback<ImportRoomKeysResult> {
+                    keysBackupService.get().restoreKeysWithRecoveryKey(
+                            currentVersion,
+                            savedKeyBackupKeyInfo?.recoveryKey ?: "",
+                            roomId,
+                            sessionId,
+                            null,
+                            it
+                    )
+                }.successfullyNumberOfImportedKeys
+            } catch (failure: Throwable) {
+                // Fail silently
+                Timber.tag(loggerTag.value).v("getFromBackup failed ${failure.localizedMessage}")
+                0
+            }
+        }
+        if (successfullyImported == 1) {
+            Timber.tag(loggerTag.value).v("Found key in backup session:$sessionId in $roomId")
+            lastFailureMap.remove(cacheKey)
+            return true
+        } else {
+            Timber.tag(loggerTag.value).v("Failed to find key in backup session:$sessionId in $roomId")
+            lastFailureMap[cacheKey] = LastTry(currentVersion.version, clock.epochMillis())
+            return false
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/RoomDecryptorProvider.kt

@@ -79,7 +79,7 @@ internal class RoomDecryptorProvider @Inject constructor(
                             newSessionListeners.toList().forEach {
                                 try {
                                     it.onNewSession(roomId, senderKey, sessionId)
-                                } catch (e: Throwable) {
+                                } catch (ignore: Throwable) {
                                 }
                             }
                         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/RoomEncryptorsStore.kt

@@ -16,12 +16,21 @@
 
 package org.matrix.android.sdk.internal.crypto
 
+import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_MEGOLM
+import org.matrix.android.sdk.api.crypto.MXCRYPTO_ALGORITHM_OLM
 import org.matrix.android.sdk.internal.crypto.algorithms.IMXEncrypting
+import org.matrix.android.sdk.internal.crypto.algorithms.megolm.MXMegolmEncryptionFactory
+import org.matrix.android.sdk.internal.crypto.algorithms.olm.MXOlmEncryptionFactory
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
 import org.matrix.android.sdk.internal.session.SessionScope
 import javax.inject.Inject
 
 @SessionScope
-internal class RoomEncryptorsStore @Inject constructor() {
+internal class RoomEncryptorsStore @Inject constructor(
+        private val cryptoStore: IMXCryptoStore,
+        private val megolmEncryptionFactory: MXMegolmEncryptionFactory,
+        private val olmEncryptionFactory: MXOlmEncryptionFactory,
+) {
 
     // MXEncrypting instance for each room.
     private val roomEncryptors = mutableMapOf<String, IMXEncrypting>()
@@ -34,7 +43,18 @@ internal class RoomEncryptorsStore @Inject constructor() {
 
     fun get(roomId: String): IMXEncrypting? {
         return synchronized(roomEncryptors) {
-            roomEncryptors[roomId]
+            val cache = roomEncryptors[roomId]
+            if (cache != null) {
+                return@synchronized cache
+            } else {
+                val alg: IMXEncrypting? = when (cryptoStore.getRoomAlgorithm(roomId)) {
+                    MXCRYPTO_ALGORITHM_MEGOLM -> megolmEncryptionFactory.create(roomId)
+                    MXCRYPTO_ALGORITHM_OLM    -> olmEncryptionFactory.create(roomId)
+                    else                      -> null
+                }
+                alg?.let { roomEncryptors.put(roomId, it) }
+                return@synchronized alg
+            }
         }
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/SecretShareManager.kt

@@ -0,0 +1,300 @@
+/*
+ * Copyright (c) 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.withContext
+import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
+import org.matrix.android.sdk.api.auth.data.Credentials
+import org.matrix.android.sdk.api.logger.LoggerTag
+import org.matrix.android.sdk.api.session.crypto.crosssigning.KEYBACKUP_SECRET_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.crosssigning.MASTER_KEY_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.crosssigning.SELF_SIGNING_KEY_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.crosssigning.USER_SIGNING_KEY_SSSS_NAME
+import org.matrix.android.sdk.api.session.crypto.keysbackup.extractCurveKeyFromRecoveryKey
+import org.matrix.android.sdk.api.session.crypto.keyshare.GossipingRequestListener
+import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
+import org.matrix.android.sdk.api.session.crypto.model.SecretShareRequest
+import org.matrix.android.sdk.api.session.events.model.Event
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.content.SecretSendEventContent
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.util.toBase64NoPadding
+import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
+import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
+import org.matrix.android.sdk.internal.crypto.tasks.createUniqueTxnId
+import org.matrix.android.sdk.internal.session.SessionScope
+import org.matrix.android.sdk.internal.util.time.Clock
+import timber.log.Timber
+import javax.inject.Inject
+
+private val loggerTag = LoggerTag("SecretShareManager", LoggerTag.CRYPTO)
+
+@SessionScope
+internal class SecretShareManager @Inject constructor(
+        private val credentials: Credentials,
+        private val cryptoStore: IMXCryptoStore,
+        private val cryptoCoroutineScope: CoroutineScope,
+        private val messageEncrypter: MessageEncrypter,
+        private val ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction,
+        private val sendToDeviceTask: SendToDeviceTask,
+        private val coroutineDispatchers: MatrixCoroutineDispatchers,
+        private val clock: Clock,
+) {
+
+    companion object {
+        private const val SECRET_SHARE_WINDOW_DURATION = 5 * 60 * 1000 // 5 minutes
+    }
+
+    /**
+     * Secret gossiping only occurs during a limited window period after interactive verification.
+     * We keep track of recent verification in memory for that purpose (no need to persist)
+     */
+    private val recentlyVerifiedDevices = mutableMapOf<String, Long>()
+    private val verifMutex = Mutex()
+
+    /**
+     * Secrets are exchanged as part of interactive verification,
+     * so we can just store in memory.
+     */
+    private val outgoingSecretRequests = mutableListOf<SecretShareRequest>()
+
+    // the listeners
+    private val gossipingRequestListeners: MutableSet<GossipingRequestListener> = HashSet()
+
+    fun addListener(listener: GossipingRequestListener) {
+        synchronized(gossipingRequestListeners) {
+            gossipingRequestListeners.add(listener)
+        }
+    }
+
+    fun removeListener(listener: GossipingRequestListener) {
+        synchronized(gossipingRequestListeners) {
+            gossipingRequestListeners.remove(listener)
+        }
+    }
+
+    /**
+     * Called when a session has been verified.
+     * This information can be used by the manager to decide whether or not to fullfill gossiping requests.
+     * This should be called as fast as possible after a successful self interactive verification
+     */
+    fun onVerificationCompleteForDevice(deviceId: String) {
+        // For now we just keep an in memory cache
+        cryptoCoroutineScope.launch {
+            verifMutex.withLock {
+                recentlyVerifiedDevices[deviceId] = clock.epochMillis()
+            }
+        }
+    }
+
+    suspend fun handleSecretRequest(toDevice: Event) {
+        val request = toDevice.getClearContent().toModel<SecretShareRequest>()
+                ?: return Unit.also {
+                    Timber.tag(loggerTag.value)
+                            .w("handleSecretRequest() : malformed request")
+                }
+
+//            val (action, requestingDeviceId, requestId, secretName) = it
+        val secretName = request.secretName ?: return Unit.also {
+            Timber.tag(loggerTag.value)
+                    .v("handleSecretRequest() : Missing secret name")
+        }
+
+        val userId = toDevice.senderId ?: return Unit.also {
+            Timber.tag(loggerTag.value)
+                    .v("handleSecretRequest() : Missing senderId")
+        }
+
+        if (userId != credentials.userId) {
+            // secrets are only shared between our own devices
+            Timber.tag(loggerTag.value)
+                    .e("Ignoring secret share request from other users $userId")
+            return
+        }
+
+        val deviceId = request.requestingDeviceId
+                ?: return Unit.also {
+                    Timber.tag(loggerTag.value)
+                            .w("handleSecretRequest() : malformed request norequestingDeviceId ")
+                }
+
+        val device = cryptoStore.getUserDevice(credentials.userId, deviceId)
+                ?: return Unit.also {
+                    Timber.tag(loggerTag.value)
+                            .e("Received secret share request from unknown device $deviceId")
+                }
+
+        val isRequestingDeviceTrusted = device.isVerified
+        val isRecentInteractiveVerification = hasBeenVerifiedLessThanFiveMinutesFromNow(device.deviceId)
+        if (isRequestingDeviceTrusted && isRecentInteractiveVerification) {
+            // we can share the secret
+
+            val secretValue = when (secretName) {
+                MASTER_KEY_SSSS_NAME       -> cryptoStore.getCrossSigningPrivateKeys()?.master
+                SELF_SIGNING_KEY_SSSS_NAME -> cryptoStore.getCrossSigningPrivateKeys()?.selfSigned
+                USER_SIGNING_KEY_SSSS_NAME -> cryptoStore.getCrossSigningPrivateKeys()?.user
+                KEYBACKUP_SECRET_SSSS_NAME -> cryptoStore.getKeyBackupRecoveryKeyInfo()?.recoveryKey
+                        ?.let {
+                            extractCurveKeyFromRecoveryKey(it)?.toBase64NoPadding()
+                        }
+                else                       -> null
+            }
+            if (secretValue == null) {
+                Timber.tag(loggerTag.value)
+                        .i("The secret is unknown $secretName, passing to app layer")
+                val toList = synchronized(gossipingRequestListeners) { gossipingRequestListeners.toList() }
+                toList.onEach { listener ->
+                    listener.onSecretShareRequest(request)
+                }
+                return
+            }
+
+            val payloadJson = mapOf(
+                    "type" to EventType.SEND_SECRET,
+                    "content" to mapOf(
+                            "request_id" to request.requestId,
+                            "secret" to secretValue
+                    )
+            )
+
+            // Is it possible that we don't have an olm session?
+            val devicesByUser = mapOf(device.userId to listOf(device))
+            val usersDeviceMap = try {
+                ensureOlmSessionsForDevicesAction.handle(devicesByUser)
+            } catch (failure: Throwable) {
+                Timber.tag(loggerTag.value)
+                        .w("Can't share secret ${request.secretName}: Failed to establish olm session")
+                return
+            }
+
+            val olmSessionResult = usersDeviceMap.getObject(device.userId, device.deviceId)
+            if (olmSessionResult?.sessionId == null) {
+                Timber.tag(loggerTag.value)
+                        .w("secret share: no session with this device $deviceId, probably because there were no one-time keys")
+                return
+            }
+
+            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(device))
+            val sendToDeviceMap = MXUsersDevicesMap<Any>()
+            sendToDeviceMap.setObject(device.userId, device.deviceId, encodedPayload)
+            val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
+            try {
+                // raise the retries for secret
+                sendToDeviceTask.executeRetry(sendToDeviceParams, 6)
+                Timber.tag(loggerTag.value)
+                        .i("successfully shared secret $secretName to ${device.shortDebugString()}")
+                // TODO add a trail for that in audit logs
+            } catch (failure: Throwable) {
+                Timber.tag(loggerTag.value)
+                        .e(failure, "failed to send shared secret $secretName to ${device.shortDebugString()}")
+            }
+        } else {
+            Timber.tag(loggerTag.value)
+                    .d(" Received secret share request from un-authorised device ${device.deviceId}")
+        }
+    }
+
+    private suspend fun hasBeenVerifiedLessThanFiveMinutesFromNow(deviceId: String): Boolean {
+        val verifTimestamp = verifMutex.withLock {
+            recentlyVerifiedDevices[deviceId]
+        } ?: return false
+
+        val age = clock.epochMillis() - verifTimestamp
+
+        return age < SECRET_SHARE_WINDOW_DURATION
+    }
+
+    suspend fun requestSecretTo(deviceId: String, secretName: String) {
+        val cryptoDeviceInfo = cryptoStore.getUserDevice(credentials.userId, deviceId) ?: return Unit.also {
+            Timber.tag(loggerTag.value)
+                    .d("Can't request secret for $secretName unknown device $deviceId")
+        }
+        val toDeviceContent = SecretShareRequest(
+                requestingDeviceId = credentials.deviceId,
+                secretName = secretName,
+                requestId = createUniqueTxnId()
+        )
+
+        verifMutex.withLock {
+            outgoingSecretRequests.add(toDeviceContent)
+        }
+
+        val contentMap = MXUsersDevicesMap<Any>()
+        contentMap.setObject(cryptoDeviceInfo.userId, cryptoDeviceInfo.deviceId, toDeviceContent)
+
+        val params = SendToDeviceTask.Params(
+                eventType = EventType.REQUEST_SECRET,
+                contentMap = contentMap
+        )
+        try {
+            withContext(coroutineDispatchers.io) {
+                sendToDeviceTask.executeRetry(params, 3)
+            }
+            Timber.tag(loggerTag.value)
+                    .d("Secret request sent for $secretName to ${cryptoDeviceInfo.shortDebugString()}")
+            // TODO update the audit trail
+        } catch (failure: Throwable) {
+            Timber.tag(loggerTag.value)
+                    .w("Failed to request secret $secretName to ${cryptoDeviceInfo.shortDebugString()}")
+        }
+    }
+
+    suspend fun onSecretSendReceived(toDevice: Event, handleGossip: ((name: String, value: String) -> Boolean)) {
+        Timber.tag(loggerTag.value)
+                .i("onSecretSend() from ${toDevice.senderId} : onSecretSendReceived ${toDevice.content?.get("sender_key")}")
+        if (!toDevice.isEncrypted()) {
+            // secret send messages must be encrypted
+            Timber.tag(loggerTag.value).e("onSecretSend() :Received unencrypted secret send event")
+            return
+        }
+
+        // Was that sent by us?
+        if (toDevice.senderId != credentials.userId) {
+            Timber.tag(loggerTag.value).e("onSecretSend() : Ignore secret from other user ${toDevice.senderId}")
+            return
+        }
+
+        val secretContent = toDevice.getClearContent().toModel<SecretSendEventContent>() ?: return
+
+        val existingRequest = verifMutex.withLock {
+            outgoingSecretRequests.firstOrNull { it.requestId == secretContent.requestId }
+        }
+
+        // As per spec:
+        // Clients should ignore m.secret.send events received from devices that it did not send an m.secret.request event to.
+        if (existingRequest?.secretName == null) {
+            Timber.tag(loggerTag.value).i("onSecretSend() : Ignore secret that was not requested: ${secretContent.requestId}")
+            return
+        }
+        // we don't need to cancel the request as we only request to one device
+        // just forget about the request now
+        verifMutex.withLock {
+            outgoingSecretRequests.remove(existingRequest)
+        }
+
+        if (!handleGossip(existingRequest.secretName, secretContent.secretValue)) {
+            // TODO Ask to application layer?
+            Timber.tag(loggerTag.value).v("onSecretSend() : secret not handled by SDK")
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/SendGossipRequestWorker.kt

@@ -1,153 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import android.content.Context
-import androidx.work.WorkerParameters
-import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.api.auth.data.Credentials
-import org.matrix.android.sdk.api.failure.shouldBeRetried
-import org.matrix.android.sdk.api.session.crypto.model.GossipingToDeviceObject
-import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingRoomKeyRequest
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyShareRequest
-import org.matrix.android.sdk.api.session.crypto.model.SecretShareRequest
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.toContent
-import org.matrix.android.sdk.internal.SessionManager
-import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.crypto.tasks.createUniqueTxnId
-import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.util.time.Clock
-import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import timber.log.Timber
-import javax.inject.Inject
-
-internal class SendGossipRequestWorker(context: Context, params: WorkerParameters, sessionManager: SessionManager) :
-        SessionSafeCoroutineWorker<SendGossipRequestWorker.Params>(context, params, sessionManager, Params::class.java) {
-
-    @JsonClass(generateAdapter = true)
-    internal data class Params(
-            override val sessionId: String,
-            val keyShareRequest: OutgoingRoomKeyRequest? = null,
-            val secretShareRequest: OutgoingSecretRequest? = null,
-            // The txnId for the sendToDevice request. Nullable for compatibility reasons, but MUST always be provided
-            // to use the same value if this worker is retried.
-            val txnId: String? = null,
-            override val lastFailureMessage: String? = null
-    ) : SessionWorkerParams
-
-    @Inject lateinit var sendToDeviceTask: SendToDeviceTask
-    @Inject lateinit var cryptoStore: IMXCryptoStore
-    @Inject lateinit var credentials: Credentials
-    @Inject lateinit var clock: Clock
-
-    override fun injectWith(injector: SessionComponent) {
-        injector.inject(this)
-    }
-
-    override suspend fun doSafeWork(params: Params): Result {
-        // params.txnId should be provided in all cases now. But Params can be deserialized by
-        // the WorkManager from data serialized in a previous version of the application, so without the txnId field.
-        // So if not present, we create a txnId
-        val txnId = params.txnId ?: createUniqueTxnId()
-        val contentMap = MXUsersDevicesMap<Any>()
-        val eventType: String
-        val requestId: String
-        when {
-            params.keyShareRequest != null    -> {
-                eventType = EventType.ROOM_KEY_REQUEST
-                requestId = params.keyShareRequest.requestId
-                val toDeviceContent = RoomKeyShareRequest(
-                        requestingDeviceId = credentials.deviceId,
-                        requestId = params.keyShareRequest.requestId,
-                        action = GossipingToDeviceObject.ACTION_SHARE_REQUEST,
-                        body = params.keyShareRequest.requestBody
-                )
-                cryptoStore.saveGossipingEvent(Event(
-                        type = eventType,
-                        content = toDeviceContent.toContent(),
-                        senderId = credentials.userId
-                ).also {
-                    it.ageLocalTs = clock.epochMillis()
-                })
-
-                params.keyShareRequest.recipients.forEach { userToDeviceMap ->
-                    userToDeviceMap.value.forEach { deviceId ->
-                        contentMap.setObject(userToDeviceMap.key, deviceId, toDeviceContent)
-                    }
-                }
-            }
-            params.secretShareRequest != null -> {
-                eventType = EventType.REQUEST_SECRET
-                requestId = params.secretShareRequest.requestId
-                val toDeviceContent = SecretShareRequest(
-                        requestingDeviceId = credentials.deviceId,
-                        requestId = params.secretShareRequest.requestId,
-                        action = GossipingToDeviceObject.ACTION_SHARE_REQUEST,
-                        secretName = params.secretShareRequest.secretName
-                )
-
-                cryptoStore.saveGossipingEvent(Event(
-                        type = eventType,
-                        content = toDeviceContent.toContent(),
-                        senderId = credentials.userId
-                ).also {
-                    it.ageLocalTs = clock.epochMillis()
-                })
-
-                params.secretShareRequest.recipients.forEach { userToDeviceMap ->
-                    userToDeviceMap.value.forEach { deviceId ->
-                        contentMap.setObject(userToDeviceMap.key, deviceId, toDeviceContent)
-                    }
-                }
-            }
-            else                              -> {
-                return buildErrorResult(params, "Unknown empty gossiping request").also {
-                    Timber.e("Unknown empty gossiping request: $params")
-                }
-            }
-        }
-        try {
-            cryptoStore.updateOutgoingGossipingRequestState(requestId, OutgoingGossipingRequestState.SENDING)
-            sendToDeviceTask.execute(
-                    SendToDeviceTask.Params(
-                            eventType = eventType,
-                            contentMap = contentMap,
-                            transactionId = txnId
-                    )
-            )
-            cryptoStore.updateOutgoingGossipingRequestState(requestId, OutgoingGossipingRequestState.SENT)
-            return Result.success()
-        } catch (throwable: Throwable) {
-            return if (throwable.shouldBeRetried()) {
-                Result.retry()
-            } else {
-                cryptoStore.updateOutgoingGossipingRequestState(requestId, OutgoingGossipingRequestState.FAILED_TO_SEND)
-                buildErrorResult(params, throwable.localizedMessage ?: "error")
-            }
-        }
-    }
-
-    override fun buildErrorParams(params: Params, message: String): Params {
-        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/SendGossipWorker.kt

@@ -1,170 +0,0 @@
-/*
- * Copyright 2020 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.crypto
-
-import android.content.Context
-import androidx.work.WorkerParameters
-import com.squareup.moshi.JsonClass
-import org.matrix.android.sdk.api.auth.data.Credentials
-import org.matrix.android.sdk.api.failure.shouldBeRetried
-import org.matrix.android.sdk.api.session.crypto.model.GossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.EventType
-import org.matrix.android.sdk.api.session.events.model.content.SecretSendEventContent
-import org.matrix.android.sdk.api.session.events.model.toContent
-import org.matrix.android.sdk.internal.SessionManager
-import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
-import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
-import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.crypto.tasks.createUniqueTxnId
-import org.matrix.android.sdk.internal.session.SessionComponent
-import org.matrix.android.sdk.internal.util.time.Clock
-import org.matrix.android.sdk.internal.worker.SessionSafeCoroutineWorker
-import org.matrix.android.sdk.internal.worker.SessionWorkerParams
-import timber.log.Timber
-import javax.inject.Inject
-
-internal class SendGossipWorker(
-        context: Context,
-        params: WorkerParameters,
-        sessionManager: SessionManager
-) : SessionSafeCoroutineWorker<SendGossipWorker.Params>(context, params, sessionManager, Params::class.java) {
-
-    @JsonClass(generateAdapter = true)
-    internal data class Params(
-            override val sessionId: String,
-            val secretValue: String,
-            val requestUserId: String?,
-            val requestDeviceId: String?,
-            val requestId: String?,
-            // The txnId for the sendToDevice request. Nullable for compatibility reasons, but MUST always be provided
-            // to use the same value if this worker is retried.
-            val txnId: String? = null,
-            override val lastFailureMessage: String? = null
-    ) : SessionWorkerParams
-
-    @Inject lateinit var sendToDeviceTask: SendToDeviceTask
-    @Inject lateinit var cryptoStore: IMXCryptoStore
-    @Inject lateinit var credentials: Credentials
-    @Inject lateinit var messageEncrypter: MessageEncrypter
-    @Inject lateinit var ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction
-    @Inject lateinit var clock: Clock
-
-    override fun injectWith(injector: SessionComponent) {
-        injector.inject(this)
-    }
-
-    override suspend fun doSafeWork(params: Params): Result {
-        // params.txnId should be provided in all cases now. But Params can be deserialized by
-        // the WorkManager from data serialized in a previous version of the application, so without the txnId field.
-        // So if not present, we create a txnId
-        val txnId = params.txnId ?: createUniqueTxnId()
-        val eventType: String = EventType.SEND_SECRET
-
-        val toDeviceContent = SecretSendEventContent(
-                requestId = params.requestId ?: "",
-                secretValue = params.secretValue
-        )
-
-        val requestingUserId = params.requestUserId ?: ""
-        val requestingDeviceId = params.requestDeviceId ?: ""
-        val deviceInfo = cryptoStore.getUserDevice(requestingUserId, requestingDeviceId)
-                ?: return buildErrorResult(params, "Unknown deviceInfo, cannot send message").also {
-                    cryptoStore.updateGossipingRequestState(
-                            requestUserId = params.requestUserId,
-                            requestDeviceId = params.requestDeviceId,
-                            requestId = params.requestId,
-                            state = GossipingRequestState.FAILED_TO_ACCEPTED
-                    )
-                    Timber.e("Unknown deviceInfo, cannot send message, sessionId: ${params.requestDeviceId}")
-                }
-
-        val sendToDeviceMap = MXUsersDevicesMap<Any>()
-
-        val devicesByUser = mapOf(requestingUserId to listOf(deviceInfo))
-        val usersDeviceMap = ensureOlmSessionsForDevicesAction.handle(devicesByUser)
-        val olmSessionResult = usersDeviceMap.getObject(requestingUserId, requestingDeviceId)
-        if (olmSessionResult?.sessionId == null) {
-            // no session with this device, probably because there
-            // were no one-time keys.
-            return buildErrorResult(params, "no session with this device").also {
-                cryptoStore.updateGossipingRequestState(
-                        requestUserId = params.requestUserId,
-                        requestDeviceId = params.requestDeviceId,
-                        requestId = params.requestId,
-                        state = GossipingRequestState.FAILED_TO_ACCEPTED
-                )
-                Timber.e("no session with this device $requestingDeviceId, probably because there were no one-time keys.")
-            }
-        }
-
-        val payloadJson = mapOf(
-                "type" to EventType.SEND_SECRET,
-                "content" to toDeviceContent.toContent()
-        )
-
-        try {
-            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
-            sendToDeviceMap.setObject(requestingUserId, requestingDeviceId, encodedPayload)
-        } catch (failure: Throwable) {
-            Timber.e("## Fail to encrypt gossip + ${failure.localizedMessage}")
-        }
-
-        cryptoStore.saveGossipingEvent(Event(
-                type = eventType,
-                content = toDeviceContent.toContent(),
-                senderId = credentials.userId
-        ).also {
-            it.ageLocalTs = clock.epochMillis()
-        })
-
-        try {
-            sendToDeviceTask.execute(
-                    SendToDeviceTask.Params(
-                            eventType = EventType.ENCRYPTED,
-                            contentMap = sendToDeviceMap,
-                            transactionId = txnId
-                    )
-            )
-            cryptoStore.updateGossipingRequestState(
-                    requestUserId = params.requestUserId,
-                    requestDeviceId = params.requestDeviceId,
-                    requestId = params.requestId,
-                    state = GossipingRequestState.ACCEPTED
-            )
-            return Result.success()
-        } catch (throwable: Throwable) {
-            return if (throwable.shouldBeRetried()) {
-                Result.retry()
-            } else {
-                cryptoStore.updateGossipingRequestState(
-                        requestUserId = params.requestUserId,
-                        requestDeviceId = params.requestDeviceId,
-                        requestId = params.requestId,
-                        state = GossipingRequestState.FAILED_TO_ACCEPTED
-                )
-                buildErrorResult(params, throwable.localizedMessage ?: "error")
-            }
-        }
-    }
-
-    override fun buildErrorParams(params: Params, message: String): Params {
-        return params.copy(lastFailureMessage = params.lastFailureMessage ?: message)
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/actions/MegolmSessionDataImporter.kt

@@ -17,12 +17,13 @@
 package org.matrix.android.sdk.internal.crypto.actions
 
 import androidx.annotation.WorkerThread
+import org.matrix.android.sdk.api.extensions.tryOrNull
 import org.matrix.android.sdk.api.listeners.ProgressListener
+import org.matrix.android.sdk.api.logger.LoggerTag
 import org.matrix.android.sdk.api.session.crypto.model.ImportRoomKeysResult
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
 import org.matrix.android.sdk.internal.crypto.MXOlmDevice
 import org.matrix.android.sdk.internal.crypto.MegolmSessionData
-import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequestManager
+import org.matrix.android.sdk.internal.crypto.OutgoingKeyRequestManager
 import org.matrix.android.sdk.internal.crypto.RoomDecryptorProvider
 import org.matrix.android.sdk.internal.crypto.algorithms.megolm.MXMegolmDecryption
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
@@ -30,12 +31,13 @@ import org.matrix.android.sdk.internal.util.time.Clock
 import timber.log.Timber
 import javax.inject.Inject
 
-internal class MegolmSessionDataImporter @Inject constructor(
+private val loggerTag = LoggerTag("MegolmSessionDataImporter", LoggerTag.CRYPTO)
-        private val olmDevice: MXOlmDevice,
+
-        private val roomDecryptorProvider: RoomDecryptorProvider,
+internal class MegolmSessionDataImporter @Inject constructor(private val olmDevice: MXOlmDevice,
-        private val outgoingGossipingRequestManager: OutgoingGossipingRequestManager,
+                                                             private val roomDecryptorProvider: RoomDecryptorProvider,
-        private val cryptoStore: IMXCryptoStore,
+                                                             private val outgoingKeyRequestManager: OutgoingKeyRequestManager,
-        private val clock: Clock,
+                                                             private val cryptoStore: IMXCryptoStore,
+                                                             private val clock: Clock,
 ) {
 
     /**
@@ -66,19 +68,23 @@ internal class MegolmSessionDataImporter @Inject constructor(
             if (null != decrypting) {
                 try {
                     val sessionId = megolmSessionData.sessionId
-                    Timber.v("## importRoomKeys retrieve senderKey " + megolmSessionData.senderKey + " sessionId " + sessionId)
+                    Timber.tag(loggerTag.value).v("## importRoomKeys retrieve senderKey ${megolmSessionData.senderKey} sessionId $sessionId")
 
                     totalNumbersOfImportedKeys++
 
                     // cancel any outstanding room key requests for this session
-                    val roomKeyRequestBody = RoomKeyRequestBody(
-                            algorithm = megolmSessionData.algorithm,
-                            roomId = megolmSessionData.roomId,
-                            senderKey = megolmSessionData.senderKey,
-                            sessionId = megolmSessionData.sessionId
-                    )
 
-                    outgoingGossipingRequestManager.cancelRoomKeyRequest(roomKeyRequestBody)
+                    Timber.tag(loggerTag.value).d("Imported megolm session $sessionId from backup=$fromBackup in ${megolmSessionData.roomId}")
+                    outgoingKeyRequestManager.postCancelRequestForSessionIfNeeded(
+                            megolmSessionData.sessionId ?: "",
+                            megolmSessionData.roomId ?: "",
+                            megolmSessionData.senderKey ?: "",
+                            tryOrNull {
+                                olmInboundGroupSessionWrappers
+                                        .firstOrNull { it.olmInboundGroupSession?.sessionIdentifier() == megolmSessionData.sessionId }
+                                        ?.firstKnownIndex?.toInt()
+                            } ?: 0
+                    )
 
                     // Have another go at decrypting events sent with this session
                     when (decrypting) {
@@ -87,7 +93,7 @@ internal class MegolmSessionDataImporter @Inject constructor(
                         }
                     }
                 } catch (e: Exception) {
-                    Timber.e(e, "## importRoomKeys() : onNewSession failed")
+                    Timber.tag(loggerTag.value).e(e, "## importRoomKeys() : onNewSession failed")
                 }
             }
 
@@ -109,7 +115,7 @@ internal class MegolmSessionDataImporter @Inject constructor(
 
         val t1 = clock.epochMillis()
 
-        Timber.v("## importMegolmSessionsData : sessions import " + (t1 - t0) + " ms (" + megolmSessionsData.size + " sessions)")
+        Timber.tag(loggerTag.value).v("## importMegolmSessionsData : sessions import " + (t1 - t0) + " ms (" + megolmSessionsData.size + " sessions)")
 
         return ImportRoomKeysResult(totalNumbersOfKeys, totalNumbersOfImportedKeys)
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/IMXDecrypting.kt

@@ -17,8 +17,6 @@
 package org.matrix.android.sdk.internal.crypto.algorithms
 
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
-import org.matrix.android.sdk.api.session.crypto.model.IncomingRoomKeyRequest
-import org.matrix.android.sdk.api.session.crypto.model.IncomingSecretShareRequest
 import org.matrix.android.sdk.api.session.crypto.model.MXEventDecryptionResult
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.internal.crypto.keysbackup.DefaultKeysBackupService
@@ -44,23 +42,4 @@ internal interface IMXDecrypting {
      * @param event the key event.
      */
     fun onRoomKeyEvent(event: Event, defaultKeysBackupService: DefaultKeysBackupService) {}
-
-    /**
-     * Determine if we have the keys necessary to respond to a room key request
-     *
-     * @param request keyRequest
-     * @return true if we have the keys and could (theoretically) share
-     */
-    fun hasKeysForKeyRequest(request: IncomingRoomKeyRequest): Boolean = false
-
-    /**
-     * Send the response to a room key request.
-     *
-     * @param request keyRequest
-     */
-    fun shareKeysWithDevice(request: IncomingRoomKeyRequest) {}
-
-    fun shareSecretWithDevice(request: IncomingSecretShareRequest, secretValue: String) {}
-
-    fun requestKeysForEvent(event: Event, withHeld: Boolean)
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/megolm/MXMegolmDecryption.kt

@@ -17,51 +17,32 @@
 package org.matrix.android.sdk.internal.crypto.algorithms.megolm
 
 import dagger.Lazy
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.sync.withLock
-import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
 import org.matrix.android.sdk.api.logger.LoggerTag
 import org.matrix.android.sdk.api.session.crypto.MXCryptoError
 import org.matrix.android.sdk.api.session.crypto.NewSessionListener
 import org.matrix.android.sdk.api.session.crypto.model.ForwardedRoomKeyContent
-import org.matrix.android.sdk.api.session.crypto.model.IncomingRoomKeyRequest
 import org.matrix.android.sdk.api.session.crypto.model.MXEventDecryptionResult
-import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
 import org.matrix.android.sdk.api.session.events.model.content.RoomKeyContent
-import org.matrix.android.sdk.api.session.events.model.content.RoomKeyWithHeldContent
 import org.matrix.android.sdk.api.session.events.model.toModel
-import org.matrix.android.sdk.internal.crypto.DeviceListManager
 import org.matrix.android.sdk.internal.crypto.MXOlmDevice
-import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequestManager
+import org.matrix.android.sdk.internal.crypto.OutgoingKeyRequestManager
-import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
-import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
 import org.matrix.android.sdk.internal.crypto.algorithms.IMXDecrypting
-import org.matrix.android.sdk.internal.crypto.algorithms.IMXWithHeldExtension
 import org.matrix.android.sdk.internal.crypto.keysbackup.DefaultKeysBackupService
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
 import org.matrix.android.sdk.internal.session.StreamEventsManager
 import timber.log.Timber
 
 private val loggerTag = LoggerTag("MXMegolmDecryption", LoggerTag.CRYPTO)
 
-internal class MXMegolmDecryption(private val userId: String,
+internal class MXMegolmDecryption(
-                                  private val olmDevice: MXOlmDevice,
+        private val olmDevice: MXOlmDevice,
-                                  private val deviceListManager: DeviceListManager,
+        private val outgoingKeyRequestManager: OutgoingKeyRequestManager,
-                                  private val outgoingGossipingRequestManager: OutgoingGossipingRequestManager,
+        private val cryptoStore: IMXCryptoStore,
-                                  private val messageEncrypter: MessageEncrypter,
+        private val liveEventManager: Lazy<StreamEventsManager>
-                                  private val ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction,
+) : IMXDecrypting {
-                                  private val cryptoStore: IMXCryptoStore,
-                                  private val sendToDeviceTask: SendToDeviceTask,
-                                  private val coroutineDispatchers: MatrixCoroutineDispatchers,
-                                  private val cryptoCoroutineScope: CoroutineScope,
-                                  private val liveEventManager: Lazy<StreamEventsManager>
-) : IMXDecrypting, IMXWithHeldExtension {
 
     var newSessionListener: NewSessionListener? = null
 
@@ -73,10 +54,7 @@ internal class MXMegolmDecryption(private val userId: String,
 
     @Throws(MXCryptoError::class)
     override suspend fun decryptEvent(event: Event, timeline: String): MXEventDecryptionResult {
-        // If cross signing is enabled, we don't send request until the keys are trusted
+        return decryptEvent(event, timeline, true)
-        // There could be a race effect here when xsigning is enabled, we should ensure that keys was downloaded once
-        val requestOnFail = cryptoStore.getMyCrossSigningInfo()?.isTrusted() == true
-        return decryptEvent(event, timeline, requestOnFail)
     }
 
     @Throws(MXCryptoError::class)
@@ -126,13 +104,14 @@ internal class MXMegolmDecryption(private val userId: String,
                             if (throwable is MXCryptoError.OlmError) {
                                 // TODO Check the value of .message
                                 if (throwable.olmException.message == "UNKNOWN_MESSAGE_INDEX") {
-                                    // addEventToPendingList(event, timeline)
+                                    // So we know that session, but it's ratcheted and we can't decrypt at that index
-                                    // The session might has been partially withheld (and only pass ratcheted)
+
+                                    if (requestKeysOnFail) {
+                                        requestKeysForEvent(event)
+                                    }
+                                    // Check if partially withheld
                                     val withHeldInfo = cryptoStore.getWithHeldMegolmSession(event.roomId, encryptedEventContent.sessionId)
                                     if (withHeldInfo != null) {
-                                        if (requestKeysOnFail) {
-                                            requestKeysForEvent(event, true)
-                                        }
                                         // Encapsulate as withHeld exception
                                         throw MXCryptoError.Base(
                                                 MXCryptoError.ErrorType.KEYS_WITHHELD,
@@ -141,10 +120,6 @@ internal class MXMegolmDecryption(private val userId: String,
                                         )
                                     }
 
-                                    if (requestKeysOnFail) {
-                                        requestKeysForEvent(event, false)
-                                    }
-
                                     throw MXCryptoError.Base(
                                             MXCryptoError.ErrorType.UNKNOWN_MESSAGE_INDEX,
                                             "UNKNOWN_MESSAGE_INDEX",
@@ -162,27 +137,22 @@ internal class MXMegolmDecryption(private val userId: String,
                                 )
                             }
                             if (throwable is MXCryptoError.Base) {
-                                if (
+                                if (throwable.errorType == MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID) {
-                                /** if the session is unknown*/
+                                    // Check if it was withheld by sender to enrich error code
-                                        throwable.errorType == MXCryptoError.ErrorType.UNKNOWN_INBOUND_SESSION_ID
-                                ) {
                                     val withHeldInfo = cryptoStore.getWithHeldMegolmSession(event.roomId, encryptedEventContent.sessionId)
                                     if (withHeldInfo != null) {
                                         if (requestKeysOnFail) {
-                                            requestKeysForEvent(event, true)
+                                            requestKeysForEvent(event)
                                         }
                                         // Encapsulate as withHeld exception
                                         throw MXCryptoError.Base(
                                                 MXCryptoError.ErrorType.KEYS_WITHHELD,
                                                 withHeldInfo.code?.value ?: "",
-                                                withHeldInfo.reason
+                                                withHeldInfo.reason)
-                                        )
+                                    }
-                                    } else {
+
-                                        // This is un-used in Matrix Android SDK2, not sure if needed
+                                    if (requestKeysOnFail) {
-                                        // addEventToPendingList(event, timeline)
+                                        requestKeysForEvent(event)
-                                        if (requestKeysOnFail) {
-                                            requestKeysForEvent(event, false)
-                                        }
                                     }
                                 }
                             }
@@ -198,55 +168,10 @@ internal class MXMegolmDecryption(private val userId: String,
      *
      * @param event the event
      */
-    override fun requestKeysForEvent(event: Event, withHeld: Boolean) {
+    private fun requestKeysForEvent(event: Event) {
-        val sender = event.senderId ?: return
+        outgoingKeyRequestManager.requestKeyForEvent(event, false)
-        val encryptedEventContent = event.content.toModel<EncryptedEventContent>()
-        val senderDevice = encryptedEventContent?.deviceId ?: return
-
-        val recipients = if (event.senderId == userId || withHeld) {
-            mapOf(
-                    userId to listOf("*")
-            )
-        } else {
-            // for the case where you share the key with a device that has a broken olm session
-            // The other user might Re-shares a megolm session key with devices if the key has already been
-            // sent to them.
-            mapOf(
-                    userId to listOf("*"),
-                    sender to listOf(senderDevice)
-            )
-        }
-
-        val requestBody = RoomKeyRequestBody(
-                roomId = event.roomId,
-                algorithm = encryptedEventContent.algorithm,
-                senderKey = encryptedEventContent.senderKey,
-                sessionId = encryptedEventContent.sessionId
-        )
-
-        outgoingGossipingRequestManager.sendRoomKeyRequest(requestBody, recipients)
     }
 
-//    /**
-//     * Add an event to the list of those we couldn't decrypt the first time we
-//     * saw them.
-//     *
-//     * @param event      the event to try to decrypt later
-//     * @param timelineId the timeline identifier
-//     */
-//    private fun addEventToPendingList(event: Event, timelineId: String) {
-//        val encryptedEventContent = event.content.toModel<EncryptedEventContent>() ?: return
-//        val pendingEventsKey = "${encryptedEventContent.senderKey}|${encryptedEventContent.sessionId}"
-//
-//        val timeline = pendingEvents.getOrPut(pendingEventsKey) { HashMap() }
-//        val events = timeline.getOrPut(timelineId) { ArrayList() }
-//
-//        if (event !in events) {
-//            Timber.v("## CRYPTO | addEventToPendingList() : add Event ${event.eventId} in room id ${event.roomId}")
-//            events.add(event)
-//        }
-//    }
-
     /**
      * Handle a key event.
      *
@@ -266,6 +191,11 @@ internal class MXMegolmDecryption(private val userId: String,
             return
         }
         if (event.getClearType() == EventType.FORWARDED_ROOM_KEY) {
+            if (!cryptoStore.isKeyGossipingEnabled()) {
+                Timber.tag(loggerTag.value)
+                        .i("onRoomKeyEvent(), ignore forward adding as per crypto config : ${roomKeyContent.roomId}|${roomKeyContent.sessionId}")
+                return
+            }
             Timber.tag(loggerTag.value).i("onRoomKeyEvent(), forward adding key : ${roomKeyContent.roomId}|${roomKeyContent.sessionId}")
             val forwardedRoomKeyContent = event.getClearContent().toModel<ForwardedRoomKeyContent>()
                     ?: return
@@ -306,7 +236,7 @@ internal class MXMegolmDecryption(private val userId: String,
         }
 
         Timber.tag(loggerTag.value).i("onRoomKeyEvent addInboundGroupSession ${roomKeyContent.sessionId}")
-        val added = olmDevice.addInboundGroupSession(
+        val addSessionResult = olmDevice.addInboundGroupSession(
                 roomKeyContent.sessionId,
                 roomKeyContent.sessionKey,
                 roomKeyContent.roomId,
@@ -316,18 +246,47 @@ internal class MXMegolmDecryption(private val userId: String,
                 exportFormat
         )
 
-        if (added) {
+        when (addSessionResult) {
+            is MXOlmDevice.AddSessionResult.Imported               -> addSessionResult.ratchetIndex
+            is MXOlmDevice.AddSessionResult.NotImportedHigherIndex -> addSessionResult.newIndex
+            else                                                   -> null
+        }?.let { index ->
+            if (event.getClearType() == EventType.FORWARDED_ROOM_KEY) {
+                val fromDevice = (event.content?.get("sender_key") as? String)?.let { senderDeviceIdentityKey ->
+                    cryptoStore.getUserDeviceList(event.senderId ?: "")
+                            ?.firstOrNull {
+                                it.identityKey() == senderDeviceIdentityKey
+                            }
+                }?.deviceId
+
+                outgoingKeyRequestManager.onRoomKeyForwarded(
+                        sessionId = roomKeyContent.sessionId,
+                        algorithm = roomKeyContent.algorithm ?: "",
+                        roomId = roomKeyContent.roomId,
+                        senderKey = senderKey,
+                        fromIndex = index,
+                        fromDevice = fromDevice,
+                        event = event)
+
+                cryptoStore.saveIncomingForwardKeyAuditTrail(
+                        roomId = roomKeyContent.roomId,
+                        sessionId = roomKeyContent.sessionId,
+                        senderKey = senderKey,
+                        algorithm = roomKeyContent.algorithm ?: "",
+                        userId = event.senderId ?: "",
+                        deviceId = fromDevice ?: "",
+                        chainIndex = index.toLong())
+
+                // The index is used to decide if we cancel sent request or if we wait for a better key
+                outgoingKeyRequestManager.postCancelRequestForSessionIfNeeded(roomKeyContent.sessionId, roomKeyContent.roomId, senderKey, index)
+            }
+        }
+
+        if (addSessionResult is MXOlmDevice.AddSessionResult.Imported) {
+            Timber.tag(loggerTag.value)
+                    .d("onRoomKeyEvent(${event.getClearType()}) : Added megolm session ${roomKeyContent.sessionId} in ${roomKeyContent.roomId}")
             defaultKeysBackupService.maybeBackupKeys()
 
-            val content = RoomKeyRequestBody(
-                    algorithm = roomKeyContent.algorithm,
-                    roomId = roomKeyContent.roomId,
-                    sessionId = roomKeyContent.sessionId,
-                    senderKey = senderKey
-            )
-
-            outgoingGossipingRequestManager.cancelRoomKeyRequest(content)
-
             onNewSession(roomKeyContent.roomId, senderKey, roomKeyContent.sessionId)
         }
     }
@@ -343,77 +302,4 @@ internal class MXMegolmDecryption(private val userId: String,
         Timber.tag(loggerTag.value).v("ON NEW SESSION $sessionId - $senderKey")
         newSessionListener?.onNewSession(roomId, senderKey, sessionId)
     }
-
-    override fun hasKeysForKeyRequest(request: IncomingRoomKeyRequest): Boolean {
-        val roomId = request.requestBody?.roomId ?: return false
-        val senderKey = request.requestBody.senderKey ?: return false
-        val sessionId = request.requestBody.sessionId ?: return false
-        return olmDevice.hasInboundSessionKeys(roomId, senderKey, sessionId)
-    }
-
-    override fun shareKeysWithDevice(request: IncomingRoomKeyRequest) {
-        // sanity checks
-        if (request.requestBody == null) {
-            return
-        }
-        val userId = request.userId ?: return
-
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-            val body = request.requestBody
-            val sessionHolder = try {
-                olmDevice.getInboundGroupSession(body.sessionId, body.senderKey, body.roomId)
-            } catch (failure: Throwable) {
-                Timber.tag(loggerTag.value).e(failure, "shareKeysWithDevice: failed to get session for request $body")
-                return@launch
-            }
-
-            val export = sessionHolder.mutex.withLock {
-                sessionHolder.wrapper.exportKeys()
-            } ?: return@launch Unit.also {
-                Timber.tag(loggerTag.value).e("shareKeysWithDevice: failed to export group session ${body.sessionId}")
-            }
-
-            runCatching { deviceListManager.downloadKeys(listOf(userId), false) }
-                    .mapCatching {
-                        val deviceId = request.deviceId
-                        val deviceInfo = cryptoStore.getUserDevice(userId, deviceId ?: "")
-                        if (deviceInfo == null) {
-                            throw RuntimeException()
-                        } else {
-                            val devicesByUser = mapOf(userId to listOf(deviceInfo))
-                            val usersDeviceMap = ensureOlmSessionsForDevicesAction.handle(devicesByUser)
-                            val olmSessionResult = usersDeviceMap.getObject(userId, deviceId)
-                            if (olmSessionResult?.sessionId == null) {
-                                // no session with this device, probably because there
-                                // were no one-time keys.
-                                Timber.tag(loggerTag.value).e("no session with this device $deviceId, probably because there were no one-time keys.")
-                                return@mapCatching
-                            }
-                            Timber.tag(loggerTag.value).i("shareKeysWithDevice() : sharing session ${body.sessionId} with device $userId:$deviceId")
-
-                            val payloadJson = mapOf(
-                                    "type" to EventType.FORWARDED_ROOM_KEY,
-                                    "content" to export
-                            )
-
-                            val encodedPayload = messageEncrypter.encryptMessage(payloadJson, listOf(deviceInfo))
-                            val sendToDeviceMap = MXUsersDevicesMap<Any>()
-                            sendToDeviceMap.setObject(userId, deviceId, encodedPayload)
-                            Timber.tag(loggerTag.value).i("shareKeysWithDevice() : sending ${body.sessionId} to $userId:$deviceId")
-                            val sendToDeviceParams = SendToDeviceTask.Params(EventType.ENCRYPTED, sendToDeviceMap)
-                            try {
-                                sendToDeviceTask.execute(sendToDeviceParams)
-                            } catch (failure: Throwable) {
-                                Timber.tag(loggerTag.value).e(failure, "shareKeysWithDevice() : Failed to send ${body.sessionId} to $userId:$deviceId")
-                            }
-                        }
-                    }
-        }
-    }
-
-    override fun onRoomKeyWithHeldEvent(withHeldInfo: RoomKeyWithHeldContent) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-            cryptoStore.addWithHeldMegolmSession(withHeldInfo)
-        }
-    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/megolm/MXMegolmDecryptionFactory.kt

@@ -17,46 +17,24 @@
 package org.matrix.android.sdk.internal.crypto.algorithms.megolm
 
 import dagger.Lazy
-import kotlinx.coroutines.CoroutineScope
-import org.matrix.android.sdk.api.MatrixCoroutineDispatchers
-import org.matrix.android.sdk.internal.crypto.DeviceListManager
 import org.matrix.android.sdk.internal.crypto.MXOlmDevice
-import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequestManager
+import org.matrix.android.sdk.internal.crypto.OutgoingKeyRequestManager
-import org.matrix.android.sdk.internal.crypto.actions.EnsureOlmSessionsForDevicesAction
-import org.matrix.android.sdk.internal.crypto.actions.MessageEncrypter
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.crypto.tasks.SendToDeviceTask
-import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.session.StreamEventsManager
 import javax.inject.Inject
 
 internal class MXMegolmDecryptionFactory @Inject constructor(
-        @UserId private val userId: String,
         private val olmDevice: MXOlmDevice,
-        private val deviceListManager: DeviceListManager,
+        private val outgoingKeyRequestManager: OutgoingKeyRequestManager,
-        private val outgoingGossipingRequestManager: OutgoingGossipingRequestManager,
-        private val messageEncrypter: MessageEncrypter,
-        private val ensureOlmSessionsForDevicesAction: EnsureOlmSessionsForDevicesAction,
         private val cryptoStore: IMXCryptoStore,
-        private val sendToDeviceTask: SendToDeviceTask,
-        private val coroutineDispatchers: MatrixCoroutineDispatchers,
-        private val cryptoCoroutineScope: CoroutineScope,
         private val eventsManager: Lazy<StreamEventsManager>
 ) {
 
     fun create(): MXMegolmDecryption {
         return MXMegolmDecryption(
-                userId,
                 olmDevice,
-                deviceListManager,
+                outgoingKeyRequestManager,
-                outgoingGossipingRequestManager,
-                messageEncrypter,
-                ensureOlmSessionsForDevicesAction,
                 cryptoStore,
-                sendToDeviceTask,
+                eventsManager)
-                coroutineDispatchers,
-                cryptoCoroutineScope,
-                eventsManager
-        )
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/megolm/MXMegolmEncryption.kt

@@ -28,7 +28,6 @@ import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
 import org.matrix.android.sdk.api.session.crypto.model.forEach
 import org.matrix.android.sdk.api.session.events.model.Content
-import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.content.RoomKeyWithHeldContent
 import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
@@ -80,7 +79,7 @@ internal class MXMegolmEncryption(
     }
 
     // Default rotation periods
-    // TODO: Make it configurable via parameters
+    // TODO Make it configurable via parameters
     // Session rotation periods
     private var sessionRotationPeriodMsgs: Int = 100
     private var sessionRotationPeriodMs: Int = 7 * 24 * 3600 * 1000
@@ -285,25 +284,14 @@ internal class MXMegolmEncryption(
         // attempted to share with) rather than the contentMap (those we did
         // share with), because we don't want to try to claim a one-time-key
         // for dead devices on every message.
-        val gossipingEventBuffer = arrayListOf<Event>()
+        for ((_, devicesToShareWith) in devicesByUser) {
-        for ((userId, devicesToShareWith) in devicesByUser) {
             for (deviceInfo in devicesToShareWith) {
                 session.sharedWithHelper.markedSessionAsShared(deviceInfo, chainIndex)
-                gossipingEventBuffer.add(
+                // XXX is it needed to add it to the audit trail?
-                        Event(
+                // For now decided that no, we are more interested by forward trail
-                                type = EventType.ROOM_KEY,
-                                senderId = myUserId,
-                                content = submap.apply {
-                                    this["session_key"] = ""
-                                    // we add a fake key for trail
-                                    this["_dest"] = "$userId|${deviceInfo.deviceId}"
-                                }
-                        ))
             }
         }
 
-        cryptoStore.saveGossipingEvents(gossipingEventBuffer)
-
         if (haveTargets) {
             t0 = clock.epochMillis()
             Timber.tag(loggerTag.value).i("shareUserDevicesKey() ${session.sessionId} : has target")
@@ -346,7 +334,8 @@ internal class MXMegolmEncryption(
                 senderKey = senderKey,
                 algorithm = MXCRYPTO_ALGORITHM_MEGOLM,
                 sessionId = sessionId,
-                codeString = code.value
+                codeString = code.value,
+                fromDevice = myDeviceId
         )
         val params = SendToDeviceTask.Params(
                 EventType.ROOM_KEY_WITHHELD,

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/olm/MXOlmDecryption.kt

@@ -265,8 +265,4 @@ internal class MXOlmDecryption(
 
         return res["payload"]
     }
-
-    override fun requestKeysForEvent(event: Event, withHeld: Boolean) {
-        // nop
-    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/algorithms/olm/MXOlmEncryption.kt

@@ -38,7 +38,7 @@ internal class MXOlmEncryption(
     override suspend fun encryptEventContent(eventContent: Content, eventType: String, userIds: List<String>): Content {
         // pick the list of recipients based on the membership list.
         //
-        // TODO: there is a race condition here! What if a new user turns up
+        // TODO there is a race condition here! What if a new user turns up
         ensureSession(userIds)
         val deviceInfos = ArrayList<CryptoDeviceInfo>()
         for (userId in userIds) {

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/crosssigning/CrossSigningOlm.kt

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto.crosssigning
+
+import org.matrix.android.sdk.api.util.JsonDict
+import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
+import org.matrix.android.sdk.internal.session.SessionScope
+import org.matrix.android.sdk.internal.util.JsonCanonicalizer
+import org.matrix.olm.OlmPkSigning
+import org.matrix.olm.OlmUtility
+import javax.inject.Inject
+
+/**
+ * Holds the OlmPkSigning for cross signing.
+ * Can be injected without having to get the full cross signing service
+ */
+@SessionScope
+internal class CrossSigningOlm @Inject constructor(
+        private val cryptoStore: IMXCryptoStore,
+) {
+
+    enum class KeyType {
+        SELF,
+        USER,
+        MASTER
+    }
+
+    var olmUtility: OlmUtility = OlmUtility()
+
+    var masterPkSigning: OlmPkSigning? = null
+    var userPkSigning: OlmPkSigning? = null
+    var selfSigningPkSigning: OlmPkSigning? = null
+
+    fun release() {
+        olmUtility.releaseUtility()
+        listOf(masterPkSigning, userPkSigning, selfSigningPkSigning).forEach { it?.releaseSigning() }
+    }
+
+    fun signObject(type: KeyType, strToSign: String): Map<String, String> {
+        val myKeys = cryptoStore.getMyCrossSigningInfo()
+        val pubKey = when (type) {
+            KeyType.SELF   -> myKeys?.selfSigningKey()
+            KeyType.USER   -> myKeys?.userKey()
+            KeyType.MASTER -> myKeys?.masterKey()
+        }?.unpaddedBase64PublicKey
+        val pkSigning = when (type) {
+            KeyType.SELF   -> selfSigningPkSigning
+            KeyType.USER   -> userPkSigning
+            KeyType.MASTER -> masterPkSigning
+        }
+        if (pubKey == null || pkSigning == null) {
+            throw Throwable("Cannot sign from this account, public and/or privateKey Unknown $type|$pkSigning")
+        }
+        val signature = pkSigning.sign(strToSign)
+        return mapOf(
+                "ed25519:$pubKey" to signature
+        )
+    }
+
+    fun verifySignature(type: KeyType, signable: JsonDict, signatures: Map<String, Map<String, String>>) {
+        val myKeys = cryptoStore.getMyCrossSigningInfo()
+                ?: throw NoSuchElementException("Cross Signing not configured")
+        val myUserID = myKeys.userId
+        val pubKey = when (type) {
+            KeyType.SELF   -> myKeys.selfSigningKey()
+            KeyType.USER   -> myKeys.userKey()
+            KeyType.MASTER -> myKeys.masterKey()
+        }?.unpaddedBase64PublicKey ?: throw NoSuchElementException("Cross Signing not configured")
+        val signaturesMadeByMyKey = signatures[myUserID] // Signatures made by me
+                ?.get("ed25519:$pubKey")
+
+        if (signaturesMadeByMyKey.isNullOrBlank()) {
+            throw IllegalArgumentException("Not signed with my key $type")
+        }
+
+        // Check that Alice USK signature of Bob MSK is valid
+        olmUtility.verifyEd25519Signature(signaturesMadeByMyKey, pubKey, JsonCanonicalizer.getCanonicalJson(Map::class.java, signable))
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/crosssigning/DefaultCrossSigningService.kt

@@ -38,6 +38,7 @@ import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 import org.matrix.android.sdk.api.util.Optional
 import org.matrix.android.sdk.api.util.fromBase64
 import org.matrix.android.sdk.internal.crypto.DeviceListManager
+import org.matrix.android.sdk.internal.crypto.OutgoingKeyRequestManager
 import org.matrix.android.sdk.internal.crypto.model.rest.UploadSignatureQueryBuilder
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
 import org.matrix.android.sdk.internal.crypto.tasks.InitializeCrossSigningTask
@@ -53,7 +54,6 @@ import org.matrix.android.sdk.internal.util.JsonCanonicalizer
 import org.matrix.android.sdk.internal.util.logLimit
 import org.matrix.android.sdk.internal.worker.WorkerParamsFactory
 import org.matrix.olm.OlmPkSigning
-import org.matrix.olm.OlmUtility
 import timber.log.Timber
 import java.util.concurrent.TimeUnit
 import javax.inject.Inject
@@ -70,19 +70,14 @@ internal class DefaultCrossSigningService @Inject constructor(
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val cryptoCoroutineScope: CoroutineScope,
         private val workManagerProvider: WorkManagerProvider,
+        private val outgoingKeyRequestManager: OutgoingKeyRequestManager,
+        private val crossSigningOlm: CrossSigningOlm,
         private val updateTrustWorkerDataRepository: UpdateTrustWorkerDataRepository
 ) : CrossSigningService,
         DeviceListManager.UserDevicesUpdateListener {
 
-    private var olmUtility: OlmUtility? = null
-
-    private var masterPkSigning: OlmPkSigning? = null
-    private var userPkSigning: OlmPkSigning? = null
-    private var selfSigningPkSigning: OlmPkSigning? = null
-
     init {
         try {
-            olmUtility = OlmUtility()
 
             // Try to get stored keys if they exist
             cryptoStore.getMyCrossSigningInfo()?.let { mxCrossSigningInfo ->
@@ -95,7 +90,7 @@ internal class DefaultCrossSigningService @Inject constructor(
                             ?.let { privateKeySeed ->
                                 val pkSigning = OlmPkSigning()
                                 if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.masterKey()?.unpaddedBase64PublicKey) {
-                                    masterPkSigning = pkSigning
+                                    crossSigningOlm.masterPkSigning = pkSigning
                                     Timber.i("## CrossSigning - Loading master key success")
                                 } else {
                                     Timber.w("## CrossSigning - Public master key does not match the private key")
@@ -108,7 +103,7 @@ internal class DefaultCrossSigningService @Inject constructor(
                             ?.let { privateKeySeed ->
                                 val pkSigning = OlmPkSigning()
                                 if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.userKey()?.unpaddedBase64PublicKey) {
-                                    userPkSigning = pkSigning
+                                    crossSigningOlm.userPkSigning = pkSigning
                                     Timber.i("## CrossSigning - Loading User Signing key success")
                                 } else {
                                     Timber.w("## CrossSigning - Public User key does not match the private key")
@@ -121,7 +116,7 @@ internal class DefaultCrossSigningService @Inject constructor(
                             ?.let { privateKeySeed ->
                                 val pkSigning = OlmPkSigning()
                                 if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.selfSigningKey()?.unpaddedBase64PublicKey) {
-                                    selfSigningPkSigning = pkSigning
+                                    crossSigningOlm.selfSigningPkSigning = pkSigning
                                     Timber.i("## CrossSigning - Loading Self Signing key success")
                                 } else {
                                     Timber.w("## CrossSigning - Public Self Signing key does not match the private key")
@@ -143,8 +138,7 @@ internal class DefaultCrossSigningService @Inject constructor(
     }
 
     fun release() {
-        olmUtility?.releaseUtility()
+        crossSigningOlm.release()
-        listOf(masterPkSigning, userPkSigning, selfSigningPkSigning).forEach { it?.releaseSigning() }
         deviceListManager.removeListener(this)
     }
 
@@ -177,9 +171,9 @@ internal class DefaultCrossSigningService @Inject constructor(
                     cryptoStore.setMyCrossSigningInfo(crossSigningInfo)
                     setUserKeysAsTrusted(userId, true)
                     cryptoStore.storePrivateKeysInfo(data.masterKeyPK, data.userKeyPK, data.selfSigningKeyPK)
-                    masterPkSigning = OlmPkSigning().apply { initWithSeed(data.masterKeyPK.fromBase64()) }
+                    crossSigningOlm.masterPkSigning = OlmPkSigning().apply { initWithSeed(data.masterKeyPK.fromBase64()) }
-                    userPkSigning = OlmPkSigning().apply { initWithSeed(data.userKeyPK.fromBase64()) }
+                    crossSigningOlm.userPkSigning = OlmPkSigning().apply { initWithSeed(data.userKeyPK.fromBase64()) }
-                    selfSigningPkSigning = OlmPkSigning().apply { initWithSeed(data.selfSigningKeyPK.fromBase64()) }
+                    crossSigningOlm.selfSigningPkSigning = OlmPkSigning().apply { initWithSeed(data.selfSigningKeyPK.fromBase64()) }
 
                     callback.onSuccess(Unit)
                 }
@@ -198,8 +192,8 @@ internal class DefaultCrossSigningService @Inject constructor(
                     val pkSigning = OlmPkSigning()
                     try {
                         if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.masterKey()?.unpaddedBase64PublicKey) {
-                            masterPkSigning?.releaseSigning()
+                            crossSigningOlm.masterPkSigning?.releaseSigning()
-                            masterPkSigning = pkSigning
+                            crossSigningOlm.masterPkSigning = pkSigning
                             Timber.i("## CrossSigning - Loading MSK success")
                             cryptoStore.storeMSKPrivateKey(mskPrivateKey)
                             return
@@ -225,8 +219,8 @@ internal class DefaultCrossSigningService @Inject constructor(
                     val pkSigning = OlmPkSigning()
                     try {
                         if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.selfSigningKey()?.unpaddedBase64PublicKey) {
-                            selfSigningPkSigning?.releaseSigning()
+                            crossSigningOlm.selfSigningPkSigning?.releaseSigning()
-                            selfSigningPkSigning = pkSigning
+                            crossSigningOlm.selfSigningPkSigning = pkSigning
                             Timber.i("## CrossSigning - Loading SSK success")
                             cryptoStore.storeSSKPrivateKey(sskPrivateKey)
                             return
@@ -252,8 +246,8 @@ internal class DefaultCrossSigningService @Inject constructor(
                     val pkSigning = OlmPkSigning()
                     try {
                         if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.userKey()?.unpaddedBase64PublicKey) {
-                            userPkSigning?.releaseSigning()
+                            crossSigningOlm.userPkSigning?.releaseSigning()
-                            userPkSigning = pkSigning
+                            crossSigningOlm.userPkSigning = pkSigning
                             Timber.i("## CrossSigning - Loading USK success")
                             cryptoStore.storeUSKPrivateKey(uskPrivateKey)
                             return
@@ -282,8 +276,8 @@ internal class DefaultCrossSigningService @Inject constructor(
                     val pkSigning = OlmPkSigning()
                     try {
                         if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.masterKey()?.unpaddedBase64PublicKey) {
-                            masterPkSigning?.releaseSigning()
+                            crossSigningOlm.masterPkSigning?.releaseSigning()
-                            masterPkSigning = pkSigning
+                            crossSigningOlm.masterPkSigning = pkSigning
                             masterKeyIsTrusted = true
                             Timber.i("## CrossSigning - Loading master key success")
                         } else {
@@ -299,8 +293,8 @@ internal class DefaultCrossSigningService @Inject constructor(
                     val pkSigning = OlmPkSigning()
                     try {
                         if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.userKey()?.unpaddedBase64PublicKey) {
-                            userPkSigning?.releaseSigning()
+                            crossSigningOlm.userPkSigning?.releaseSigning()
-                            userPkSigning = pkSigning
+                            crossSigningOlm.userPkSigning = pkSigning
                             userKeyIsTrusted = true
                             Timber.i("## CrossSigning - Loading master key success")
                         } else {
@@ -316,8 +310,8 @@ internal class DefaultCrossSigningService @Inject constructor(
                     val pkSigning = OlmPkSigning()
                     try {
                         if (pkSigning.initWithSeed(privateKeySeed) == mxCrossSigningInfo.selfSigningKey()?.unpaddedBase64PublicKey) {
-                            selfSigningPkSigning?.releaseSigning()
+                            crossSigningOlm.selfSigningPkSigning?.releaseSigning()
-                            selfSigningPkSigning = pkSigning
+                            crossSigningOlm.selfSigningPkSigning = pkSigning
                             selfSignedKeyIsTrusted = true
                             Timber.i("## CrossSigning - Loading master key success")
                         } else {
@@ -405,7 +399,11 @@ internal class DefaultCrossSigningService @Inject constructor(
 
         // Check that Alice USK signature of Bob MSK is valid
         try {
-            olmUtility!!.verifyEd25519Signature(masterKeySignaturesMadeByMyUserKey, myUserKey.unpaddedBase64PublicKey, otherMasterKey.canonicalSignable())
+            crossSigningOlm.olmUtility.verifyEd25519Signature(
+                    masterKeySignaturesMadeByMyUserKey,
+                    myUserKey.unpaddedBase64PublicKey,
+                    otherMasterKey.canonicalSignable()
+            )
         } catch (failure: Throwable) {
             return UserTrustResult.InvalidSignature(myUserKey, masterKeySignaturesMadeByMyUserKey)
         }
@@ -459,7 +457,7 @@ internal class DefaultCrossSigningService @Inject constructor(
                 if (potentialDevice != null && potentialDevice.isVerified) {
                     // Check signature validity?
                     try {
-                        olmUtility?.verifyEd25519Signature(value, potentialDevice.fingerprint(), myMasterKey.canonicalSignable())
+                        crossSigningOlm.olmUtility.verifyEd25519Signature(value, potentialDevice.fingerprint(), myMasterKey.canonicalSignable())
                         isMaterKeyTrusted = true
                         return@forEach
                     } catch (failure: Throwable) {
@@ -488,7 +486,11 @@ internal class DefaultCrossSigningService @Inject constructor(
 
         // Check that Alice USK signature of Alice MSK is valid
         try {
-            olmUtility!!.verifyEd25519Signature(userKeySignaturesMadeByMyMasterKey, myMasterKey.unpaddedBase64PublicKey, myUserKey.canonicalSignable())
+            crossSigningOlm.olmUtility.verifyEd25519Signature(
+                    userKeySignaturesMadeByMyMasterKey,
+                    myMasterKey.unpaddedBase64PublicKey,
+                    myUserKey.canonicalSignable()
+            )
         } catch (failure: Throwable) {
             return UserTrustResult.InvalidSignature(myUserKey, userKeySignaturesMadeByMyMasterKey)
         }
@@ -507,7 +509,11 @@ internal class DefaultCrossSigningService @Inject constructor(
 
         // Check that Alice USK signature of Alice MSK is valid
         try {
-            olmUtility!!.verifyEd25519Signature(ssKeySignaturesMadeByMyMasterKey, myMasterKey.unpaddedBase64PublicKey, mySSKey.canonicalSignable())
+            crossSigningOlm.olmUtility.verifyEd25519Signature(
+                    ssKeySignaturesMadeByMyMasterKey,
+                    myMasterKey.unpaddedBase64PublicKey,
+                    mySSKey.canonicalSignable()
+            )
         } catch (failure: Throwable) {
             return UserTrustResult.InvalidSignature(mySSKey, ssKeySignaturesMadeByMyMasterKey)
         }
@@ -560,7 +566,7 @@ internal class DefaultCrossSigningService @Inject constructor(
                 return@launch
             }
             val userPubKey = myKeys.userKey()?.unpaddedBase64PublicKey
-            if (userPubKey == null || userPkSigning == null) {
+            if (userPubKey == null || crossSigningOlm.userPkSigning == null) {
                 callback.onFailure(Throwable("## CrossSigning - Cannot sign from this account, privateKeyUnknown $userPubKey"))
                 return@launch
             }
@@ -569,7 +575,7 @@ internal class DefaultCrossSigningService @Inject constructor(
             val newSignature = JsonCanonicalizer.getCanonicalJson(
                     Map::class.java,
                     otherMasterKeys.signalableJSONDictionary()
-            ).let { userPkSigning?.sign(it) }
+            ).let { crossSigningOlm.userPkSigning?.sign(it) }
 
             if (newSignature == null) {
                 // race??
@@ -616,13 +622,13 @@ internal class DefaultCrossSigningService @Inject constructor(
             }
 
             val ssPubKey = myKeys.selfSigningKey()?.unpaddedBase64PublicKey
-            if (ssPubKey == null || selfSigningPkSigning == null) {
+            if (ssPubKey == null || crossSigningOlm.selfSigningPkSigning == null) {
                 callback.onFailure(Throwable("Cannot sign from this account, public and/or privateKey Unknown $ssPubKey"))
                 return@launch
             }
 
             // Sign with self signing
-            val newSignature = selfSigningPkSigning?.sign(device.canonicalSignable())
+            val newSignature = crossSigningOlm.selfSigningPkSigning?.sign(device.canonicalSignable())
 
             if (newSignature == null) {
                 // race??
@@ -695,7 +701,11 @@ internal class DefaultCrossSigningService @Inject constructor(
 
         // Check  bob's device is signed by bob's SSK
         try {
-            olmUtility!!.verifyEd25519Signature(otherSSKSignature, otherKeys.selfSigningKey()?.unpaddedBase64PublicKey, otherDevice.canonicalSignable())
+            crossSigningOlm.olmUtility.verifyEd25519Signature(
+                    otherSSKSignature,
+                    otherKeys.selfSigningKey()?.unpaddedBase64PublicKey,
+                    otherDevice.canonicalSignable()
+            )
         } catch (e: Throwable) {
             return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.InvalidDeviceSignature(otherDeviceId, otherSSKSignature, e))
         }
@@ -745,7 +755,11 @@ internal class DefaultCrossSigningService @Inject constructor(
 
         // Check  bob's device is signed by bob's SSK
         try {
-            olmUtility!!.verifyEd25519Signature(otherSSKSignature, otherKeys.selfSigningKey()?.unpaddedBase64PublicKey, otherDevice.canonicalSignable())
+            crossSigningOlm.olmUtility.verifyEd25519Signature(
+                    otherSSKSignature,
+                    otherKeys.selfSigningKey()?.unpaddedBase64PublicKey,
+                    otherDevice.canonicalSignable()
+            )
         } catch (e: Throwable) {
             return legacyFallbackTrust(locallyTrusted, DeviceTrustResult.InvalidDeviceSignature(otherDevice.deviceId, otherSSKSignature, e))
         }
@@ -785,7 +799,8 @@ internal class DefaultCrossSigningService @Inject constructor(
         // If it's me, recheck trust of all users and devices?
         val users = ArrayList<String>()
         if (otherUserId == userId && currentTrust != trusted) {
-//                reRequestAllPendingRoomKeyRequest()
+            // notify key requester
+            outgoingKeyRequestManager.onSelfCrossSigningTrustChanged(trusted)
             cryptoStore.updateUsersTrust {
                 users.add(it)
                 checkUserTrust(it).isVerified()
@@ -800,19 +815,4 @@ internal class DefaultCrossSigningService @Inject constructor(
             }
         }
     }
-
-//    private fun reRequestAllPendingRoomKeyRequest() {
-//        cryptoCoroutineScope.launch(coroutineDispatchers.crypto) {
-//            Timber.d("## CrossSigning - reRequest pending outgoing room key requests")
-//            cryptoStore.getOutgoingRoomKeyRequests().forEach {
-//                it.requestBody?.let { requestBody ->
-//                    if (cryptoStore.getInboundGroupSession(requestBody.sessionId ?: "", requestBody.senderKey ?: "") == null) {
-//                        outgoingRoomKeyRequestManager.resendRoomKeyRequest(requestBody)
-//                    } else {
-//                        outgoingRoomKeyRequestManager.cancelRoomKeyRequest(requestBody)
-//                    }
-//                }
-//            }
-//        }
-//    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/keysbackup/DefaultKeysBackupService.kt

@@ -54,6 +54,7 @@ import org.matrix.android.sdk.internal.crypto.MXOlmDevice
 import org.matrix.android.sdk.internal.crypto.MegolmSessionData
 import org.matrix.android.sdk.internal.crypto.ObjectSigner
 import org.matrix.android.sdk.internal.crypto.actions.MegolmSessionDataImporter
+import org.matrix.android.sdk.internal.crypto.crosssigning.CrossSigningOlm
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.SignalableMegolmBackupAuthData
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.BackupKeysResult
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.CreateKeysBackupVersionBody
@@ -63,16 +64,11 @@ import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.RoomKeysBack
 import org.matrix.android.sdk.internal.crypto.keysbackup.model.rest.UpdateKeysBackupVersionBody
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.CreateKeysBackupVersionTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.DeleteBackupTask
-import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.DeleteRoomSessionDataTask
-import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.DeleteRoomSessionsDataTask
-import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.DeleteSessionsDataTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.GetKeysBackupLastVersionTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.GetKeysBackupVersionTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.GetRoomSessionDataTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.GetRoomSessionsDataTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.GetSessionsDataTask
-import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.StoreRoomSessionDataTask
-import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.StoreRoomSessionsDataTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.StoreSessionsDataTask
 import org.matrix.android.sdk.internal.crypto.keysbackup.tasks.UpdateKeysBackupVersionTask
 import org.matrix.android.sdk.internal.crypto.model.OlmInboundGroupSessionWrapper2
@@ -107,21 +103,17 @@ internal class DefaultKeysBackupService @Inject constructor(
         private val cryptoStore: IMXCryptoStore,
         private val olmDevice: MXOlmDevice,
         private val objectSigner: ObjectSigner,
+        private val crossSigningOlm: CrossSigningOlm,
         // Actions
         private val megolmSessionDataImporter: MegolmSessionDataImporter,
         // Tasks
         private val createKeysBackupVersionTask: CreateKeysBackupVersionTask,
         private val deleteBackupTask: DeleteBackupTask,
-        private val deleteRoomSessionDataTask: DeleteRoomSessionDataTask,
-        private val deleteRoomSessionsDataTask: DeleteRoomSessionsDataTask,
-        private val deleteSessionDataTask: DeleteSessionsDataTask,
         private val getKeysBackupLastVersionTask: GetKeysBackupLastVersionTask,
         private val getKeysBackupVersionTask: GetKeysBackupVersionTask,
         private val getRoomSessionDataTask: GetRoomSessionDataTask,
         private val getRoomSessionsDataTask: GetRoomSessionsDataTask,
         private val getSessionsDataTask: GetSessionsDataTask,
-        private val storeRoomSessionDataTask: StoreRoomSessionDataTask,
-        private val storeSessionsDataTask: StoreRoomSessionsDataTask,
         private val storeSessionDataTask: StoreSessionsDataTask,
         private val updateKeysBackupVersionTask: UpdateKeysBackupVersionTask,
         // Task executor
@@ -168,58 +160,76 @@ internal class DefaultKeysBackupService @Inject constructor(
     override fun prepareKeysBackupVersion(password: String?,
                                           progressListener: ProgressListener?,
                                           callback: MatrixCallback<MegolmBackupCreationInfo>) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
-            runCatching {
+            try {
-                withContext(coroutineDispatchers.crypto) {
+                val olmPkDecryption = OlmPkDecryption()
-                    val olmPkDecryption = OlmPkDecryption()
+                val signalableMegolmBackupAuthData = if (password != null) {
-                    val signalableMegolmBackupAuthData = if (password != null) {
+                    // Generate a private key from the password
-                        // Generate a private key from the password
+                    val backgroundProgressListener = if (progressListener == null) {
-                        val backgroundProgressListener = if (progressListener == null) {
+                        null
-                            null
+                    } else {
-                        } else {
+                        object : ProgressListener {
-                            object : ProgressListener {
+                            override fun onProgress(progress: Int, total: Int) {
-                                override fun onProgress(progress: Int, total: Int) {
+                                uiHandler.post {
-                                    uiHandler.post {
+                                    try {
-                                        try {
+                                        progressListener.onProgress(progress, total)
-                                            progressListener.onProgress(progress, total)
+                                    } catch (e: Exception) {
-                                        } catch (e: Exception) {
+                                        Timber.e(e, "prepareKeysBackupVersion: onProgress failure")
-                                            Timber.e(e, "prepareKeysBackupVersion: onProgress failure")
-                                        }
                                     }
                                 }
                             }
                         }
-
-                        val generatePrivateKeyResult = generatePrivateKeyWithPassword(password, backgroundProgressListener)
-                        SignalableMegolmBackupAuthData(
-                                publicKey = olmPkDecryption.setPrivateKey(generatePrivateKeyResult.privateKey),
-                                privateKeySalt = generatePrivateKeyResult.salt,
-                                privateKeyIterations = generatePrivateKeyResult.iterations
-                        )
-                    } else {
-                        val publicKey = olmPkDecryption.generateKey()
-
-                        SignalableMegolmBackupAuthData(
-                                publicKey = publicKey
-                        )
                     }
-
+                    val generatePrivateKeyResult = generatePrivateKeyWithPassword(password, backgroundProgressListener)
-                    val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, signalableMegolmBackupAuthData.signalableJSONDictionary())
+                    SignalableMegolmBackupAuthData(
-
+                            publicKey = olmPkDecryption.setPrivateKey(generatePrivateKeyResult.privateKey),
-                    val signedMegolmBackupAuthData = MegolmBackupAuthData(
+                            privateKeySalt = generatePrivateKeyResult.salt,
-                            publicKey = signalableMegolmBackupAuthData.publicKey,
+                            privateKeyIterations = generatePrivateKeyResult.iterations
-                            privateKeySalt = signalableMegolmBackupAuthData.privateKeySalt,
-                            privateKeyIterations = signalableMegolmBackupAuthData.privateKeyIterations,
-                            signatures = objectSigner.signObject(canonicalJson)
                     )
-
+                } else {
-                    MegolmBackupCreationInfo(
+                    val publicKey = olmPkDecryption.generateKey()
-                            algorithm = MXCRYPTO_ALGORITHM_MEGOLM_BACKUP,
+                    SignalableMegolmBackupAuthData(
-                            authData = signedMegolmBackupAuthData,
+                            publicKey = publicKey
-                            recoveryKey = computeRecoveryKey(olmPkDecryption.privateKey())
                     )
                 }
-            }.foldToCallback(callback)
+
+                val canonicalJson = JsonCanonicalizer.getCanonicalJson(Map::class.java, signalableMegolmBackupAuthData.signalableJSONDictionary())
+
+                val signatures = mutableMapOf<String, MutableMap<String, String>>()
+
+                val deviceSignature = objectSigner.signObject(canonicalJson)
+                deviceSignature.forEach { (userID, content) ->
+                    signatures[userID] = content.toMutableMap()
+                }
+
+                // If we have cross signing add signature, will throw if cross signing not properly configured
+                try {
+                    val crossSign = crossSigningOlm.signObject(CrossSigningOlm.KeyType.MASTER, canonicalJson)
+                    signatures[credentials.userId]?.putAll(crossSign)
+                } catch (failure: Throwable) {
+                    // ignore and log
+                    Timber.w(failure, "prepareKeysBackupVersion: failed to sign with cross signing keys")
+                }
+
+                val signedMegolmBackupAuthData = MegolmBackupAuthData(
+                        publicKey = signalableMegolmBackupAuthData.publicKey,
+                        privateKeySalt = signalableMegolmBackupAuthData.privateKeySalt,
+                        privateKeyIterations = signalableMegolmBackupAuthData.privateKeyIterations,
+                        signatures = signatures
+                )
+                val creationInfo = MegolmBackupCreationInfo(
+                        algorithm = MXCRYPTO_ALGORITHM_MEGOLM_BACKUP,
+                        authData = signedMegolmBackupAuthData,
+                        recoveryKey = computeRecoveryKey(olmPkDecryption.privateKey())
+                )
+                uiHandler.post {
+                    callback.onSuccess(creationInfo)
+                }
+            } catch (failure: Throwable) {
+                uiHandler.post {
+                    callback.onFailure(failure)
+                }
+            }
         }
     }
 
@@ -267,41 +277,39 @@ internal class DefaultKeysBackupService @Inject constructor(
     }
 
     override fun deleteBackup(version: String, callback: MatrixCallback<Unit>?) {
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
-            withContext(coroutineDispatchers.crypto) {
+            // If we're currently backing up to this backup... stop.
-                // If we're currently backing up to this backup... stop.
+            // (We start using it automatically in createKeysBackupVersion so this is symmetrical).
-                // (We start using it automatically in createKeysBackupVersion so this is symmetrical).
+            if (keysBackupVersion != null && version == keysBackupVersion?.version) {
-                if (keysBackupVersion != null && version == keysBackupVersion?.version) {
+                resetKeysBackupData()
-                    resetKeysBackupData()
+                keysBackupVersion = null
-                    keysBackupVersion = null
+                keysBackupStateManager.state = KeysBackupState.Unknown
-                    keysBackupStateManager.state = KeysBackupState.Unknown
+            }
-                }
 
-                deleteBackupTask
+            deleteBackupTask
-                        .configureWith(DeleteBackupTask.Params(version)) {
+                    .configureWith(DeleteBackupTask.Params(version)) {
-                            this.callback = object : MatrixCallback<Unit> {
+                        this.callback = object : MatrixCallback<Unit> {
-                                private fun eventuallyRestartBackup() {
+                            private fun eventuallyRestartBackup() {
-                                    // Do not stay in KeysBackupState.Unknown but check what is available on the homeserver
+                                // Do not stay in KeysBackupState.Unknown but check what is available on the homeserver
-                                    if (state == KeysBackupState.Unknown) {
+                                if (state == KeysBackupState.Unknown) {
-                                        checkAndStartKeysBackup()
+                                    checkAndStartKeysBackup()
-                                    }
-                                }
-
-                                override fun onSuccess(data: Unit) {
-                                    eventuallyRestartBackup()
-
-                                    uiHandler.post { callback?.onSuccess(Unit) }
-                                }
-
-                                override fun onFailure(failure: Throwable) {
-                                    eventuallyRestartBackup()
-
-                                    uiHandler.post { callback?.onFailure(failure) }
                                 }
                             }
+
+                            override fun onSuccess(data: Unit) {
+                                eventuallyRestartBackup()
+
+                                uiHandler.post { callback?.onSuccess(Unit) }
+                            }
+
+                            override fun onFailure(failure: Throwable) {
+                                eventuallyRestartBackup()
+
+                                uiHandler.post { callback?.onFailure(failure) }
+                            }
                         }
-                        .executeBy(taskExecutor)
+                    }
-            }
+                    .executeBy(taskExecutor)
         }
     }
 
@@ -339,6 +347,10 @@ internal class DefaultKeysBackupService @Inject constructor(
 
     override fun backupAllGroupSessions(progressListener: ProgressListener?,
                                         callback: MatrixCallback<Unit>?) {
+        if (!isEnabled || backupOlmPkEncryption == null || keysBackupVersion == null) {
+            callback?.onFailure(Throwable("Backup not enabled"))
+            return
+        }
         // Get a status right now
         getBackupProgress(object : ProgressListener {
             override fun onProgress(progress: Int, total: Int) {
@@ -427,18 +439,41 @@ internal class DefaultKeysBackupService @Inject constructor(
 
         for ((keyId, mySignature) in mySigs) {
             // XXX: is this how we're supposed to get the device id?
-            var deviceId: String? = null
+            var deviceOrCrossSigningKeyId: String? = null
             val components = keyId.split(":")
             if (components.size == 2) {
-                deviceId = components[1]
+                deviceOrCrossSigningKeyId = components[1]
             }
 
-            if (deviceId != null) {
+            // Let's check if it's my master key
-                val device = cryptoStore.getUserDevice(userId, deviceId)
+            val myMSKPKey = cryptoStore.getMyCrossSigningInfo()?.masterKey()?.unpaddedBase64PublicKey
+            if (deviceOrCrossSigningKeyId == myMSKPKey) {
+                // we have to check if we can trust
+
+                var isSignatureValid = false
+                try {
+                    crossSigningOlm.verifySignature(CrossSigningOlm.KeyType.MASTER, authData.signalableJSONDictionary(), authData.signatures)
+                    isSignatureValid = true
+                } catch (failure: Throwable) {
+                    Timber.w(failure, "getKeysBackupTrust: Bad signature from my user MSK")
+                }
+                val mskTrusted = cryptoStore.getMyCrossSigningInfo()?.masterKey()?.trustLevel?.isVerified() == true
+                if (isSignatureValid && mskTrusted) {
+                    keysBackupVersionTrustIsUsable = true
+                }
+                val signature = KeysBackupVersionTrustSignature.UserSignature(
+                        keyId = deviceOrCrossSigningKeyId,
+                        cryptoCrossSigningKey = cryptoStore.getMyCrossSigningInfo()?.masterKey(),
+                        valid = isSignatureValid
+                )
+
+                keysBackupVersionTrustSignatures.add(signature)
+            } else if (deviceOrCrossSigningKeyId != null) {
+                val device = cryptoStore.getUserDevice(userId, deviceOrCrossSigningKeyId)
                 var isSignatureValid = false
 
                 if (device == null) {
-                    Timber.v("getKeysBackupTrust: Signature from unknown device $deviceId")
+                    Timber.v("getKeysBackupTrust: Signature from unknown device $deviceOrCrossSigningKeyId")
                 } else {
                     val fingerprint = device.fingerprint()
                     if (fingerprint != null) {
@@ -455,8 +490,8 @@ internal class DefaultKeysBackupService @Inject constructor(
                     }
                 }
 
-                val signature = KeysBackupVersionTrustSignature(
+                val signature = KeysBackupVersionTrustSignature.DeviceSignature(
-                        deviceId = deviceId,
+                        deviceId = deviceOrCrossSigningKeyId,
                         device = device,
                         valid = isSignatureValid,
                 )
@@ -480,10 +515,11 @@ internal class DefaultKeysBackupService @Inject constructor(
 
         if (authData == null) {
             Timber.w("trustKeyBackupVersion:trust: Key backup is missing required data")
-
+            uiHandler.post {
-            callback.onFailure(IllegalArgumentException("Missing element"))
+                callback.onFailure(IllegalArgumentException("Missing element"))
+            }
         } else {
-            cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+            cryptoCoroutineScope.launch(coroutineDispatchers.io) {
                 val updateKeysBackupVersionBody = withContext(coroutineDispatchers.crypto) {
                     // Get current signatures, or create an empty set
                     val myUserSignatures = authData.signatures?.get(userId).orEmpty().toMutableMap()
@@ -536,11 +572,15 @@ internal class DefaultKeysBackupService @Inject constructor(
 
                                     checkAndStartWithKeysBackupVersion(newKeysBackupVersion)
 
-                                    callback.onSuccess(data)
+                                    uiHandler.post {
+                                        callback.onSuccess(data)
+                                    }
                                 }
 
                                 override fun onFailure(failure: Throwable) {
-                                    callback.onFailure(failure)
+                                    uiHandler.post {
+                                        callback.onFailure(failure)
+                                    }
                                 }
                             }
                         }
@@ -554,15 +594,14 @@ internal class DefaultKeysBackupService @Inject constructor(
                                                        callback: MatrixCallback<Unit>) {
         Timber.v("trustKeysBackupVersionWithRecoveryKey: version ${keysBackupVersion.version}")
 
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
-            val isValid = withContext(coroutineDispatchers.crypto) {
+            val isValid = isValidRecoveryKeyForKeysBackupVersion(recoveryKey, keysBackupVersion)
-                isValidRecoveryKeyForKeysBackupVersion(recoveryKey, keysBackupVersion)
-            }
 
             if (!isValid) {
                 Timber.w("trustKeyBackupVersionWithRecoveryKey: Invalid recovery key.")
-
+                uiHandler.post {
-                callback.onFailure(IllegalArgumentException("Invalid recovery key or password"))
+                    callback.onFailure(IllegalArgumentException("Invalid recovery key or password"))
+                }
             } else {
                 trustKeysBackupVersion(keysBackupVersion, true, callback)
             }
@@ -574,15 +613,14 @@ internal class DefaultKeysBackupService @Inject constructor(
                                                       callback: MatrixCallback<Unit>) {
         Timber.v("trustKeysBackupVersionWithPassphrase: version ${keysBackupVersion.version}")
 
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
-            val recoveryKey = withContext(coroutineDispatchers.crypto) {
+            val recoveryKey = recoveryKeyFromPassword(password, keysBackupVersion, null)
-                recoveryKeyFromPassword(password, keysBackupVersion, null)
-            }
 
             if (recoveryKey == null) {
                 Timber.w("trustKeysBackupVersionWithPassphrase: Key backup is missing required data")
-
+                uiHandler.post {
-                callback.onFailure(IllegalArgumentException("Missing element"))
+                    callback.onFailure(IllegalArgumentException("Missing element"))
+                }
             } else {
                 // Check trust using the recovery key
                 trustKeysBackupVersionWithRecoveryKey(keysBackupVersion, recoveryKey, callback)
@@ -593,30 +631,28 @@ internal class DefaultKeysBackupService @Inject constructor(
     override fun onSecretKeyGossip(secret: String) {
         Timber.i("## CrossSigning - onSecretKeyGossip")
 
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
             try {
-                when (val keysBackupLastVersionResult = getKeysBackupLastVersionTask.execute(Unit)) {
+                val keysBackupVersion = getKeysBackupLastVersionTask.execute(Unit).toKeysVersionResult()
-                    KeysBackupLastVersionResult.NoKeysBackup  -> {
+                        ?: return@launch Unit.also {
-                        Timber.d("No keys backup found")
+                            Timber.d("Failed to get backup last version")
-                    }
-                    is KeysBackupLastVersionResult.KeysBackup -> {
-                        val keysBackupVersion = keysBackupLastVersionResult.keysVersionResult
-                        val recoveryKey = computeRecoveryKey(secret.fromBase64())
-                        if (isValidRecoveryKeyForKeysBackupVersion(recoveryKey, keysBackupVersion)) {
-                            awaitCallback<Unit> {
-                                trustKeysBackupVersion(keysBackupVersion, true, it)
-                            }
-                            val importResult = awaitCallback<ImportRoomKeysResult> {
-                                restoreKeysWithRecoveryKey(keysBackupVersion, recoveryKey, null, null, null, it)
-                            }
-                            withContext(coroutineDispatchers.crypto) {
-                                cryptoStore.saveBackupRecoveryKey(recoveryKey, keysBackupVersion.version)
-                            }
-                            Timber.i("onSecretKeyGossip: Recovered keys $importResult")
-                        } else {
-                            Timber.e("onSecretKeyGossip: Recovery key is not valid ${keysBackupVersion.version}")
                         }
+                val recoveryKey = computeRecoveryKey(secret.fromBase64())
+                if (isValidRecoveryKeyForKeysBackupVersion(recoveryKey, keysBackupVersion)) {
+                    awaitCallback<Unit> {
+                        trustKeysBackupVersion(keysBackupVersion, true, it)
                     }
+                    // we don't want to start immediately downloading all as it can take very long
+
+//                    val importResult = awaitCallback<ImportRoomKeysResult> {
+//                        restoreKeysWithRecoveryKey(keysBackupVersion, recoveryKey, null, null, null, it)
+//                    }
+                    withContext(coroutineDispatchers.crypto) {
+                        cryptoStore.saveBackupRecoveryKey(recoveryKey, keysBackupVersion.version)
+                    }
+                    Timber.i("onSecretKeyGossip: saved valid backup key")
+                } else {
+                    Timber.e("onSecretKeyGossip: Recovery key is not valid ${keysBackupVersion.version}")
                 }
             } catch (failure: Throwable) {
                 Timber.e("onSecretKeyGossip: failed to trust key backup version ${keysBackupVersion?.version}")
@@ -679,9 +715,9 @@ internal class DefaultKeysBackupService @Inject constructor(
                                             callback: MatrixCallback<ImportRoomKeysResult>) {
         Timber.v("restoreKeysWithRecoveryKey: From backup version: ${keysVersionResult.version}")
 
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
             runCatching {
-                val decryption = withContext(coroutineDispatchers.crypto) {
+                val decryption = withContext(coroutineDispatchers.computation) {
                     // Check if the recovery is valid before going any further
                     if (!isValidRecoveryKeyForKeysBackupVersion(recoveryKey, keysVersionResult)) {
                         Timber.e("restoreKeysWithRecoveryKey: Invalid recovery key for this keys version")
@@ -754,7 +790,19 @@ internal class DefaultKeysBackupService @Inject constructor(
                     }
                     result
                 }
-            }.foldToCallback(callback)
+            }.foldToCallback(object : MatrixCallback<ImportRoomKeysResult> {
+                override fun onSuccess(data: ImportRoomKeysResult) {
+                    uiHandler.post {
+                        callback.onSuccess(data)
+                    }
+                }
+
+                override fun onFailure(failure: Throwable) {
+                    uiHandler.post {
+                        callback.onFailure(failure)
+                    }
+                }
+            })
         }
     }
 
@@ -766,7 +814,7 @@ internal class DefaultKeysBackupService @Inject constructor(
                                               callback: MatrixCallback<ImportRoomKeysResult>) {
         Timber.v("[MXKeyBackup] restoreKeyBackup with password: From backup version: ${keysBackupVersion.version}")
 
-        cryptoCoroutineScope.launch(coroutineDispatchers.main) {
+        cryptoCoroutineScope.launch(coroutineDispatchers.io) {
             runCatching {
                 val progressListener = if (stepProgressListener != null) {
                     object : ProgressListener {
@@ -791,7 +839,19 @@ internal class DefaultKeysBackupService @Inject constructor(
                         restoreKeysWithRecoveryKey(keysBackupVersion, recoveryKey, roomId, sessionId, stepProgressListener, it)
                     }
                 }
-            }.foldToCallback(callback)
+            }.foldToCallback(object : MatrixCallback<ImportRoomKeysResult> {
+                override fun onSuccess(data: ImportRoomKeysResult) {
+                    uiHandler.post {
+                        callback.onSuccess(data)
+                    }
+                }
+
+                override fun onFailure(failure: Throwable) {
+                    uiHandler.post {
+                        callback.onFailure(failure)
+                    }
+                }
+            })
         }
     }
 
@@ -817,12 +877,16 @@ internal class DefaultKeysBackupService @Inject constructor(
             )
         } else if (roomId != null) {
             // Get all keys for the room
-            val data = getRoomSessionsDataTask.execute(GetRoomSessionsDataTask.Params(roomId, version))
+            val data = withContext(coroutineDispatchers.io) {
+                getRoomSessionsDataTask.execute(GetRoomSessionsDataTask.Params(roomId, version))
+            }
             // Convert to KeysBackupData
             KeysBackupData(mutableMapOf(roomId to data))
         } else {
             // Get all keys
-            getSessionsDataTask.execute(GetSessionsDataTask.Params(version))
+            withContext(coroutineDispatchers.io) {
+                getSessionsDataTask.execute(GetSessionsDataTask.Params(version))
+            }
         }
     }
 
@@ -1198,7 +1262,6 @@ internal class DefaultKeysBackupService @Inject constructor(
             Timber.v("backupKeys: Invalid configuration")
             backupAllGroupSessionsCallback?.onFailure(IllegalStateException("Invalid configuration"))
             resetBackupAllGroupSessionsListeners()
-
             return
         }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/repository/WarnOnUnknownDeviceRepository.kt

@@ -22,7 +22,7 @@ import javax.inject.Inject
 @SessionScope
 internal class WarnOnUnknownDeviceRepository @Inject constructor() {
 
-    // TODO: set it back to true by default. Need UI
+    // TODO set it back to true by default. Need UI
     // Warn the user if some new devices are detected while encrypting a message.
     private var warnOnUnknownDevices = false
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/secrets/DefaultSharedSecretStorageService.kt

@@ -40,7 +40,7 @@ import org.matrix.android.sdk.api.session.securestorage.SsssKeySpec
 import org.matrix.android.sdk.api.session.securestorage.SsssPassphrase
 import org.matrix.android.sdk.api.util.fromBase64
 import org.matrix.android.sdk.api.util.toBase64NoPadding
-import org.matrix.android.sdk.internal.crypto.OutgoingGossipingRequestManager
+import org.matrix.android.sdk.internal.crypto.SecretShareManager
 import org.matrix.android.sdk.internal.crypto.keysbackup.generatePrivateKeyWithPassword
 import org.matrix.android.sdk.internal.crypto.tools.HkdfSha256
 import org.matrix.android.sdk.internal.crypto.tools.withOlmDecryption
@@ -57,7 +57,7 @@ import kotlin.experimental.and
 internal class DefaultSharedSecretStorageService @Inject constructor(
         @UserId private val userId: String,
         private val accountDataService: SessionAccountDataService,
-        private val outgoingGossipingRequestManager: OutgoingGossipingRequestManager,
+        private val secretShareManager: SecretShareManager,
         private val coroutineDispatchers: MatrixCoroutineDispatchers,
         private val cryptoCoroutineScope: CoroutineScope
 ) : SharedSecretStorageService {
@@ -380,10 +380,7 @@ internal class DefaultSharedSecretStorageService @Inject constructor(
         return IntegrityResult.Success(keyInfo.content.passphrase != null)
     }
 
-    override fun requestSecret(name: String, myOtherDeviceId: String) {
+    override suspend fun requestSecret(name: String, myOtherDeviceId: String) {
-        outgoingGossipingRequestManager.sendSecretShareRequest(
+        secretShareManager.requestSecretTo(myOtherDeviceId, name)
-                name,
-                mapOf(userId to listOf(myOtherDeviceId))
-        )
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/IMXCryptoStore.kt

@@ -19,23 +19,22 @@ package org.matrix.android.sdk.internal.crypto.store
 import androidx.lifecycle.LiveData
 import androidx.paging.PagedList
 import org.matrix.android.sdk.api.session.crypto.NewSessionListener
+import org.matrix.android.sdk.api.session.crypto.OutgoingKeyRequest
+import org.matrix.android.sdk.api.session.crypto.OutgoingRoomKeyRequestState
 import org.matrix.android.sdk.api.session.crypto.crosssigning.CryptoCrossSigningKey
 import org.matrix.android.sdk.api.session.crypto.crosssigning.MXCrossSigningInfo
 import org.matrix.android.sdk.api.session.crypto.crosssigning.PrivateKeysInfo
 import org.matrix.android.sdk.api.session.crypto.keysbackup.SavedKeyBackupKeyInfo
+import org.matrix.android.sdk.api.session.crypto.model.AuditTrail
 import org.matrix.android.sdk.api.session.crypto.model.CryptoDeviceInfo
 import org.matrix.android.sdk.api.session.crypto.model.DeviceInfo
-import org.matrix.android.sdk.api.session.crypto.model.GossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.IncomingRoomKeyRequest
 import org.matrix.android.sdk.api.session.crypto.model.MXUsersDevicesMap
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingGossipingRequestState
-import org.matrix.android.sdk.api.session.crypto.model.OutgoingRoomKeyRequest
 import org.matrix.android.sdk.api.session.crypto.model.RoomKeyRequestBody
+import org.matrix.android.sdk.api.session.crypto.model.TrailType
 import org.matrix.android.sdk.api.session.events.model.Event
 import org.matrix.android.sdk.api.session.events.model.content.RoomKeyWithHeldContent
+import org.matrix.android.sdk.api.session.events.model.content.WithHeldCode
 import org.matrix.android.sdk.api.util.Optional
-import org.matrix.android.sdk.internal.crypto.IncomingShareRequestCommon
-import org.matrix.android.sdk.internal.crypto.OutgoingSecretRequest
 import org.matrix.android.sdk.internal.crypto.model.OlmInboundGroupSessionWrapper2
 import org.matrix.android.sdk.internal.crypto.model.OlmSessionWrapper
 import org.matrix.android.sdk.internal.crypto.model.OutboundGroupSessionWrapper
@@ -82,6 +81,15 @@ internal interface IMXCryptoStore {
      */
     fun setGlobalBlacklistUnverifiedDevices(block: Boolean)
 
+    /**
+     * Enable or disable key gossiping.
+     * Default is true.
+     * If set to false this device won't send key_request nor will accept key forwarded
+     */
+    fun enableKeyGossiping(enable: Boolean)
+
+    fun isKeyGossipingEnabled(): Boolean
+
     /**
      * Provides the rooms ids list in which the messages are not encrypted for the unverified devices.
      *
@@ -125,18 +133,6 @@ internal interface IMXCryptoStore {
      */
     fun getDeviceTrackingStatuses(): Map<String, Int>
 
-    /**
-     * @return the pending IncomingRoomKeyRequest requests
-     */
-    fun getPendingIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
-
-    fun getPendingIncomingGossipingRequests(): List<IncomingShareRequestCommon>
-
-    fun storeIncomingGossipingRequest(request: IncomingShareRequestCommon, ageLocalTS: Long?)
-
-    fun storeIncomingGossipingRequests(requests: List<IncomingShareRequestCommon>)
-//    fun getPendingIncomingSecretShareRequests(): List<IncomingSecretShareRequest>
-
     /**
      * Indicate if the store contains data for the passed account.
      *
@@ -377,7 +373,9 @@ internal interface IMXCryptoStore {
      * @param requestBody the request body
      * @return an OutgoingRoomKeyRequest instance or null
      */
-    fun getOutgoingRoomKeyRequest(requestBody: RoomKeyRequestBody): OutgoingRoomKeyRequest?
+    fun getOutgoingRoomKeyRequest(requestBody: RoomKeyRequestBody): OutgoingKeyRequest?
+    fun getOutgoingRoomKeyRequest(requestId: String): OutgoingKeyRequest?
+    fun getOutgoingRoomKeyRequest(roomId: String, sessionId: String, algorithm: String, senderKey: String): List<OutgoingKeyRequest>
 
     /**
      * Look for an existing outgoing room key request, and if none is found, add a new one.
@@ -385,39 +383,59 @@ internal interface IMXCryptoStore {
      * @param request the request
      * @return either the same instance as passed in, or the existing one.
      */
-    fun getOrAddOutgoingRoomKeyRequest(requestBody: RoomKeyRequestBody, recipients: Map<String, List<String>>): OutgoingRoomKeyRequest?
+    fun getOrAddOutgoingRoomKeyRequest(requestBody: RoomKeyRequestBody, recipients: Map<String, List<String>>, fromIndex: Int): OutgoingKeyRequest
+    fun updateOutgoingRoomKeyRequestState(requestId: String, newState: OutgoingRoomKeyRequestState)
+    fun updateOutgoingRoomKeyRequiredIndex(requestId: String, newIndex: Int)
+    fun updateOutgoingRoomKeyReply(
+            roomId: String,
+            sessionId: String,
+            algorithm: String,
+            senderKey: String,
+            fromDevice: String?,
+            event: Event)
 
-    fun getOrAddOutgoingSecretShareRequest(secretName: String, recipients: Map<String, List<String>>): OutgoingSecretRequest?
+    fun deleteOutgoingRoomKeyRequest(requestId: String)
+    fun deleteOutgoingRoomKeyRequestInState(state: OutgoingRoomKeyRequestState)
 
-    fun saveGossipingEvent(event: Event) = saveGossipingEvents(listOf(event))
+    fun saveIncomingKeyRequestAuditTrail(
+            requestId: String,
+            roomId: String,
+            sessionId: String,
+            senderKey: String,
+            algorithm: String,
+            fromUser: String,
+            fromDevice: String
+    )
 
-    fun saveGossipingEvents(events: List<Event>)
+    fun saveWithheldAuditTrail(
+            roomId: String,
+            sessionId: String,
+            senderKey: String,
+            algorithm: String,
+            code: WithHeldCode,
+            userId: String,
+            deviceId: String
+    )
 
-    fun updateGossipingRequestState(request: IncomingShareRequestCommon, state: GossipingRequestState) {
+    fun saveForwardKeyAuditTrail(
-        updateGossipingRequestState(
+            roomId: String,
-                requestUserId = request.userId,
+            sessionId: String,
-                requestDeviceId = request.deviceId,
+            senderKey: String,
-                requestId = request.requestId,
+            algorithm: String,
-                state = state
+            userId: String,
-        )
+            deviceId: String,
-    }
+            chainIndex: Long?
+    )
 
-    fun updateGossipingRequestState(requestUserId: String?,
+    fun saveIncomingForwardKeyAuditTrail(
-                                    requestDeviceId: String?,
+            roomId: String,
-                                    requestId: String?,
+            sessionId: String,
-                                    state: GossipingRequestState)
+            senderKey: String,
-
+            algorithm: String,
-    /**
+            userId: String,
-     * Search an IncomingRoomKeyRequest
+            deviceId: String,
-     *
+            chainIndex: Long?
-     * @param userId    the user id
+    )
-     * @param deviceId  the device id
-     * @param requestId the request id
-     * @return an IncomingRoomKeyRequest if it exists, else null
-     */
-    fun getIncomingRoomKeyRequest(userId: String, deviceId: String, requestId: String): IncomingRoomKeyRequest?
-
-    fun updateOutgoingGossipingRequestState(requestId: String, state: OutgoingGossipingRequestState)
 
     fun addNewSessionListener(listener: NewSessionListener)
 
@@ -477,17 +495,15 @@ internal interface IMXCryptoStore {
     fun getSharedWithInfo(roomId: String?, sessionId: String): MXUsersDevicesMap<Int>
     // Dev tools
 
-    fun getOutgoingRoomKeyRequests(): List<OutgoingRoomKeyRequest>
+    fun getOutgoingRoomKeyRequests(): List<OutgoingKeyRequest>
-    fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingRoomKeyRequest>>
+    fun getOutgoingRoomKeyRequestsPaged(): LiveData<PagedList<OutgoingKeyRequest>>
-    fun getOutgoingSecretKeyRequests(): List<OutgoingSecretRequest>
+    fun getGossipingEventsTrail(): LiveData<PagedList<AuditTrail>>
-    fun getOutgoingSecretRequest(secretName: String): OutgoingSecretRequest?
+    fun <T> getGossipingEventsTrail(type: TrailType, mapper: ((AuditTrail) -> T)): LiveData<PagedList<T>>
-    fun getIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest>
+    fun getGossipingEvents(): List<AuditTrail>
-    fun getIncomingRoomKeyRequestsPaged(): LiveData<PagedList<IncomingRoomKeyRequest>>
-    fun getGossipingEventsTrail(): LiveData<PagedList<Event>>
-    fun getGossipingEvents(): List<Event>
 
     fun setDeviceKeysUploaded(uploaded: Boolean)
     fun areDeviceKeysUploaded(): Boolean
     fun tidyUpDataBase()
     fun logDbUsageInfo()
+    fun getOutgoingRoomKeyRequests(inStates: Set<OutgoingRoomKeyRequestState>): List<OutgoingKeyRequest>
 }