Merge pull request #939 from matrix-org/rav/get_devices_api

GET /devices endpoint

synapse/handlers/device.py

@@ -69,3 +69,30 @@ class DeviceHandler(BaseHandler):
                 attempts += 1
 
         raise StoreError(500, "Couldn't generate a device ID.")
+
+    @defer.inlineCallbacks
+    def get_devices_by_user(self, user_id):
+        """
+        Retrieve the given user's devices
+
+        Args:
+            user_id (str):
+        Returns:
+            defer.Deferred: dict[str, dict[str, X]]: map from device_id to
+            info on the device
+        """
+
+        devices = yield self.store.get_devices_by_user(user_id)
+
+        ips = yield self.store.get_last_client_ip_by_device(
+            devices=((user_id, device_id) for device_id in devices.keys())
+        )
+
+        for device_id in devices.keys():
+            ip = ips.get((user_id, device_id), {})
+            devices[device_id].update({
+                "last_seen_ts": ip.get("last_seen"),
+                "last_seen_ip": ip.get("ip"),
+            })
+
+        defer.returnValue(devices)

synapse/rest/__init__.py

@@ -46,6 +46,7 @@ from synapse.rest.client.v2_alpha import (
     account_data,
     report_event,
     openid,
+    devices,
 )
 
 from synapse.http.server import JsonResource
@@ -90,3 +91,4 @@ class ClientRestResource(JsonResource):
         account_data.register_servlets(hs, client_resource)
         report_event.register_servlets(hs, client_resource)
         openid.register_servlets(hs, client_resource)
+        devices.register_servlets(hs, client_resource)

synapse/rest/client/v2_alpha/_base.py

@@ -25,7 +25,9 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def client_v2_patterns(path_regex, releases=(0,)):
+def client_v2_patterns(path_regex, releases=(0,),
+                       v2_alpha=True,
+                       unstable=True):
     """Creates a regex compiled client path with the correct client path
     prefix.
 
@@ -35,9 +37,12 @@ def client_v2_patterns(path_regex, releases=(0,)):
     Returns:
         SRE_Pattern
     """
-    patterns = [re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex)]
+    patterns = []
-    unstable_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/unstable")
+    if v2_alpha:
-    patterns.append(re.compile("^" + unstable_prefix + path_regex))
+        patterns.append(re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex))
+    if unstable:
+        unstable_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/unstable")
+        patterns.append(re.compile("^" + unstable_prefix + path_regex))
     for release in releases:
         new_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/r%d" % release)
         patterns.append(re.compile("^" + new_prefix + path_regex))

synapse/rest/client/v2_alpha/devices.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_patterns
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class DevicesRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/devices$", releases=[], v2_alpha=False)
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(DevicesRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.device_handler = hs.get_device_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        devices = yield self.device_handler.get_devices_by_user(
+            requester.user.to_string()
+        )
+        defer.returnValue((200, {"devices": devices}))
+
+
+def register_servlets(hs, http_server):
+    DevicesRestServlet(hs).register(http_server)

synapse/storage/client_ips.py

@@ -13,10 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
 from ._base import SQLBaseStore, Cache
 
 from twisted.internet import defer
 
+logger = logging.getLogger(__name__)
 
 # Number of msec of granularity to store the user IP 'last seen' time. Smaller
 # times give more inserts into the database even for readonly API hits
@@ -67,3 +70,72 @@ class ClientIpStore(SQLBaseStore):
             desc="insert_client_ip",
             lock=False,
         )
+
+    @defer.inlineCallbacks
+    def get_last_client_ip_by_device(self, devices):
+        """For each device_id listed, give the user_ip it was last seen on
+
+        Args:
+            devices (iterable[(str, str)]):  list of (user_id, device_id) pairs
+
+        Returns:
+            defer.Deferred: resolves to a dict, where the keys
+            are (user_id, device_id) tuples. The values are also dicts, with
+            keys giving the column names
+        """
+
+        res = yield self.runInteraction(
+            "get_last_client_ip_by_device",
+            self._get_last_client_ip_by_device_txn,
+            retcols=(
+                "user_id",
+                "access_token",
+                "ip",
+                "user_agent",
+                "device_id",
+                "last_seen",
+            ),
+            devices=devices
+        )
+
+        ret = {(d["user_id"], d["device_id"]): d for d in res}
+        defer.returnValue(ret)
+
+    @classmethod
+    def _get_last_client_ip_by_device_txn(cls, txn, devices, retcols):
+        def where_clause_for_device(d):
+                return
+
+        where_clauses = []
+        bindings = []
+        for (user_id, device_id) in devices:
+            if device_id is None:
+                where_clauses.append("(user_id = ? AND device_id IS NULL)")
+                bindings.extend((user_id, ))
+            else:
+                where_clauses.append("(user_id = ? AND device_id = ?)")
+                bindings.extend((user_id, device_id))
+
+        inner_select = (
+            "SELECT MAX(last_seen) mls, user_id, device_id FROM user_ips "
+            "WHERE %(where)s "
+            "GROUP BY user_id, device_id"
+        ) % {
+            "where": " OR ".join(where_clauses),
+        }
+
+        sql = (
+            "SELECT %(retcols)s FROM user_ips "
+            "JOIN (%(inner_select)s) ips ON"
+            "    user_ips.last_seen = ips.mls AND"
+            "    user_ips.user_id = ips.user_id AND"
+            "    (user_ips.device_id = ips.device_id OR"
+            "         (user_ips.device_id IS NULL AND ips.device_id IS NULL)"
+            "    )"
+        ) % {
+            "retcols": ",".join("user_ips." + c for c in retcols),
+            "inner_select": inner_select,
+        }
+
+        txn.execute(sql, bindings)
+        return cls.cursor_to_dict(txn)

synapse/storage/devices.py

@@ -65,7 +65,7 @@ class DeviceStore(SQLBaseStore):
             user_id (str): The ID of the user which owns the device
             device_id (str): The ID of the device to retrieve
         Returns:
-            defer.Deferred for a namedtuple containing the device information
+            defer.Deferred for a dict containing the device information
         Raises:
             StoreError: if the device is not found
         """
@@ -75,3 +75,23 @@ class DeviceStore(SQLBaseStore):
             retcols=("user_id", "device_id", "display_name"),
             desc="get_device",
         )
+
+    @defer.inlineCallbacks
+    def get_devices_by_user(self, user_id):
+        """Retrieve all of a user's registered devices.
+
+        Args:
+            user_id (str):
+        Returns:
+            defer.Deferred: resolves to a dict from device_id to a dict
+            containing "device_id", "user_id" and "display_name" for each
+            device.
+        """
+        devices = yield self._simple_select_list(
+            table="devices",
+            keyvalues={"user_id": user_id},
+            retcols=("user_id", "device_id", "display_name"),
+            desc="get_devices_by_user"
+        )
+
+        defer.returnValue({d["device_id"]: d for d in devices})

synapse/storage/schema/delta/33/user_ips_index.sql

@@ -0,0 +1,16 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE INDEX user_ips_device_id ON user_ips(user_id, device_id, last_seen);

tests/handlers/test_device.py

@@ -12,25 +12,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from synapse import types
 from twisted.internet import defer
 
-from synapse.handlers.device import DeviceHandler
+import synapse.handlers.device
-from tests import unittest
+import synapse.storage
-from tests.utils import setup_test_homeserver
+from tests import unittest, utils
-
-
-class DeviceHandlers(object):
-    def __init__(self, hs):
-        self.device_handler = DeviceHandler(hs)
 
 
 class DeviceTestCase(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(DeviceTestCase, self).__init__(*args, **kwargs)
+        self.store = None    # type: synapse.storage.DataStore
+        self.handler = None  # type: device.DeviceHandler
+        self.clock = None    # type: utils.MockClock
+
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver(handlers=None)
+        hs = yield utils.setup_test_homeserver(handlers=None)
-        self.hs.handlers = handlers = DeviceHandlers(self.hs)
+        self.handler = synapse.handlers.device.DeviceHandler(hs)
-        self.handler = handlers.device_handler
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
 
     @defer.inlineCallbacks
     def test_device_is_created_if_doesnt_exist(self):
@@ -73,3 +75,55 @@ class DeviceTestCase(unittest.TestCase):
 
         dev = yield self.handler.store.get_device("theresa", device_id)
         self.assertEqual(dev["display_name"], "display")
+
+    @defer.inlineCallbacks
+    def test_get_devices_by_user(self):
+        # check this works for both devices which have a recorded client_ip,
+        # and those which don't.
+        user1 = "@boris:aaa"
+        user2 = "@theresa:bbb"
+        yield self._record_user(user1, "xyz", "display 0")
+        yield self._record_user(user1, "fco", "display 1", "token1", "ip1")
+        yield self._record_user(user1, "abc", "display 2", "token2", "ip2")
+        yield self._record_user(user1, "abc", "display 2", "token3", "ip3")
+
+        yield self._record_user(user2, "def", "dispkay", "token4", "ip4")
+
+        res = yield self.handler.get_devices_by_user(user1)
+        self.assertEqual(3, len(res.keys()))
+        self.assertDictContainsSubset({
+            "user_id": user1,
+            "device_id": "xyz",
+            "display_name": "display 0",
+            "last_seen_ip": None,
+            "last_seen_ts": None,
+        }, res["xyz"])
+        self.assertDictContainsSubset({
+            "user_id": user1,
+            "device_id": "fco",
+            "display_name": "display 1",
+            "last_seen_ip": "ip1",
+            "last_seen_ts": 1000000,
+        }, res["fco"])
+        self.assertDictContainsSubset({
+            "user_id": user1,
+            "device_id": "abc",
+            "display_name": "display 2",
+            "last_seen_ip": "ip3",
+            "last_seen_ts": 3000000,
+        }, res["abc"])
+
+    @defer.inlineCallbacks
+    def _record_user(self, user_id, device_id, display_name,
+                     access_token=None, ip=None):
+        device_id = yield self.handler.check_device_registered(
+            user_id=user_id,
+            device_id=device_id,
+            initial_device_display_name=display_name
+        )
+
+        if ip is not None:
+            yield self.store.insert_client_ip(
+                types.UserID.from_string(user_id),
+                access_token, ip, "user_agent", device_id)
+            self.clock.advance_time(1000)

tests/storage/test_client_ips.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import synapse.server
+import synapse.storage
+import synapse.types
+import tests.unittest
+import tests.utils
+
+
+class ClientIpStoreTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(ClientIpStoreTestCase, self).__init__(*args, **kwargs)
+        self.store = None  # type: synapse.storage.DataStore
+        self.clock = None  # type: tests.utils.MockClock
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+
+    @defer.inlineCallbacks
+    def test_insert_new_client_ip(self):
+        self.clock.now = 12345678
+        user_id = "@user:id"
+        yield self.store.insert_client_ip(
+            synapse.types.UserID.from_string(user_id),
+            "access_token", "ip", "user_agent", "device_id",
+        )
+
+        # deliberately use an iterable here to make sure that the lookup
+        # method doesn't iterate it twice
+        device_list = iter(((user_id, "device_id"),))
+        result = yield self.store.get_last_client_ip_by_device(device_list)
+
+        r = result[(user_id, "device_id")]
+        self.assertDictContainsSubset(
+            {
+                "user_id": user_id,
+                "device_id": "device_id",
+                "access_token": "access_token",
+                "ip": "ip",
+                "user_agent": "user_agent",
+                "last_seen": 12345678000,
+            },
+            r
+        )

tests/storage/test_devices.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import tests.unittest
+import tests.utils
+
+
+class DeviceStoreTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(DeviceStoreTestCase, self).__init__(*args, **kwargs)
+        self.store = None  # type: synapse.storage.DataStore
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def test_store_new_device(self):
+        yield self.store.store_device(
+            "user_id", "device_id", "display_name"
+        )
+
+        res = yield self.store.get_device("user_id", "device_id")
+        self.assertDictContainsSubset({
+            "user_id": "user_id",
+            "device_id": "device_id",
+            "display_name": "display_name",
+        }, res)
+
+    @defer.inlineCallbacks
+    def test_get_devices_by_user(self):
+        yield self.store.store_device(
+            "user_id", "device1", "display_name 1"
+        )
+        yield self.store.store_device(
+            "user_id", "device2", "display_name 2"
+        )
+        yield self.store.store_device(
+            "user_id2", "device3", "display_name 3"
+        )
+
+        res = yield self.store.get_devices_by_user("user_id")
+        self.assertEqual(2, len(res.keys()))
+        self.assertDictContainsSubset({
+            "user_id": "user_id",
+            "device_id": "device1",
+            "display_name": "display_name 1",
+        }, res["device1"])
+        self.assertDictContainsSubset({
+            "user_id": "user_id",
+            "device_id": "device2",
+            "display_name": "display_name 2",
+        }, res["device2"])