Merge pull request #1109 from matrix-org/erikj/partial_indices

Add WHERE clause support to index creation

synapse/storage/background_updates.py

@@ -219,7 +219,7 @@ class BackgroundUpdateStore(SQLBaseStore):
         self._background_update_handlers[update_name] = update_handler
 
     def register_background_index_update(self, update_name, index_name,
-                                         table, columns):
+                                         table, columns, where_clause=None):
         """Helper for store classes to do a background index addition
 
         To use:
@@ -243,13 +243,19 @@ class BackgroundUpdateStore(SQLBaseStore):
             conc = True
         else:
             conc = False
+            # We don't use partial indices on SQLite as it wasn't introduced
+            # until 3.8, and wheezy has 3.7
+            where_clause = None
 
-        sql = "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)" \
+        sql = (
-              % {
+            "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)"
+            " %(where_clause)s"
+        ) % {
             "conc": "CONCURRENTLY" if conc else "",
             "name": index_name,
             "table": table,
             "columns": ", ".join(columns),
+            "where_clause": "WHERE " + where_clause if where_clause else ""
         }
 
         def create_index_concurrently(conn):

synapse/storage/event_push_actions.py

@@ -36,7 +36,7 @@ class EventPushActionsStore(SQLBaseStore):
             self.EPA_HIGHLIGHT_INDEX,
             index_name="event_push_actions_u_highlight",
             table="event_push_actions",
-            columns=["user_id", "stream_ordering", "highlight"],
+            columns=["user_id", "stream_ordering"],
         )
 
     def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):

synapse/storage/events.py

@@ -189,6 +189,14 @@ class EventsStore(SQLBaseStore):
             self._background_reindex_fields_sender,
         )
 
+        self.register_background_index_update(
+            "event_contains_url_index",
+            index_name="event_contains_url_index",
+            table="events",
+            columns=["room_id", "topological_ordering", "stream_ordering"],
+            where_clause="contains_url = true AND outlier = false",
+        )
+
         self._event_persist_queue = _EventPeristenceQueue()
 
     def persist_events(self, events_and_contexts, backfilled=False):

synapse/storage/schema/delta/35/contains_url.sql

@@ -0,0 +1,17 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ INSERT into background_updates (update_name, progress_json)
+     VALUES ('event_contains_url_index', '{}');