From 719aec406487c9eb7942ce8c6a12a046218e93e2 Mon Sep 17 00:00:00 2001 From: Sean Enck Date: Fri, 21 Apr 2017 11:03:32 -0400 Subject: [PATCH 1/8] clarify metric setup to use 'scrape_configs' section of yaml and use an array for target --- docs/metrics-howto.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 7390ab85c9..143cd0f42f 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -21,13 +21,12 @@ How to monitor Synapse metrics using Prometheus 3. Add a prometheus target for synapse. - It needs to set the ``metrics_path`` to a non-default value:: + It needs to set the ``metrics_path`` to a non-default value (under ``scrape_configs``):: - job_name: "synapse" metrics_path: "/_synapse/metrics" static_configs: - - targets: - "my.server.here:9092" + - targets: ["my.server.here:9092"] If your prometheus is older than 1.5.2, you will need to replace ``static_configs`` in the above with ``target_groups``. From 22f3d3ae769a031c42cda1fb233dd91e28e585d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Apr 2017 11:43:03 +0100 Subject: [PATCH 2/8] Reduce _get_state_group_for_event cache size --- synapse/storage/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 927a936013..e89001d994 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -507,7 +507,7 @@ class StateStore(SQLBaseStore): state_map = yield self.get_state_ids_for_events([event_id], types) defer.returnValue(state_map[event_id]) - @cached(num_args=2, max_entries=100000) + @cached(num_args=2, max_entries=50000) def _get_state_group_for_event(self, room_id, event_id): return self._simple_select_one_onecol( table="event_to_state_groups", From d9aa645f86db733bb0419b5f5428ba9a9c799735 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Apr 2017 14:38:51 +0100 Subject: [PATCH 3/8] Reduce size of joined_user cache The _get_joined_users_from_context cache stores a mapping from user_id to avatar_url and display_name. Instead of storing those in a dict, store them in a namedtuple as that uses much less memory. We also try converting the string to ascii to further reduce the size. --- synapse/push/bulk_push_rule_evaluator.py | 7 +++++-- synapse/rest/client/v1/room.py | 8 +++++++- synapse/storage/roommember.py | 22 ++++++++++++++-------- synapse/util/stringutils.py | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 78b095c903..503fef4261 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -87,8 +87,11 @@ class BulkPushRuleEvaluator: condition_cache = {} for uid, rules in self.rules_by_user.items(): - display_name = room_members.get(uid, {}).get("display_name", None) - if not display_name: + display_name = None + profile_info = room_members.get(uid, {}) + if profile_info: + display_name = profile_info.display_name + else: # Handle the case where we are pushing a membership event to # that user, as they might not be already joined. if event.type == EventTypes.Member and event.state_key == uid: diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 0bdd6b5b36..c376ab8fd7 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -406,7 +406,13 @@ class JoinedRoomMemberListRestServlet(ClientV1RestServlet): users_with_profile = yield self.state.get_current_user_in_room(room_id) defer.returnValue((200, { - "joined": users_with_profile + "joined": { + user_id: { + "avatar_url": profile.avatar_url, + "display_name": profile.display_name, + } + for user_id, profile in users_with_profile.iteritems() + } })) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 367dbbbcf6..68e724ac7f 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -19,6 +19,7 @@ from collections import namedtuple from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from synapse.util.stringutils import to_ascii from synapse.api.constants import Membership, EventTypes from synapse.types import get_domain_from_id @@ -35,6 +36,11 @@ RoomsForUser = namedtuple( ) +ProfileInfo = namedtuple( + "ProfileInfo", ("avatar_url", "display_name") +) + + _MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update" @@ -422,20 +428,20 @@ class RoomMemberStore(SQLBaseStore): ) users_in_room = { - row["user_id"]: { - "display_name": row["display_name"], - "avatar_url": row["avatar_url"], - } + to_ascii(row["user_id"]): ProfileInfo( + avatar_url=to_ascii(row["avatar_url"]), + display_name=to_ascii(row["display_name"]), + ) for row in rows } if event is not None and event.type == EventTypes.Member: if event.membership == Membership.JOIN: if event.event_id in member_event_ids: - users_in_room[event.state_key] = { - "display_name": event.content.get("displayname", None), - "avatar_url": event.content.get("avatar_url", None), - } + users_in_room[to_ascii(event.state_key)] = ProfileInfo( + display_name=to_ascii(event.content.get("displayname", None)), + avatar_url=to_ascii(event.content.get("avatar_url", None)), + ) defer.returnValue(users_in_room) diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index a100f151d4..95a6168e16 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -40,3 +40,17 @@ def is_ascii(s): return False else: return True + + +def to_ascii(s): + """Converts a string to ascii if it is ascii, otherwise leave it alone. + + If given None then will return None. + """ + if s is None: + return None + + try: + return s.encode("ascii") + except UnicodeEncodeError: + return s From f144365281e0c925ea467669cbfc6253e00f10e6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Apr 2017 15:18:26 +0100 Subject: [PATCH 4/8] Comment --- synapse/storage/roommember.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 68e724ac7f..7ad2198d96 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -36,6 +36,8 @@ RoomsForUser = namedtuple( ) +# We store this using a namedtuple so that we save about 3x space over using a +# dict. ProfileInfo = namedtuple( "ProfileInfo", ("avatar_url", "display_name") ) From f7181615f2264edcfd5aa135df127d88a2b16a1f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Apr 2017 15:22:59 +0100 Subject: [PATCH 5/8] Don't specify default as dict --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 503fef4261..e40f62de78 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -88,7 +88,7 @@ class BulkPushRuleEvaluator: for uid, rules in self.rules_by_user.items(): display_name = None - profile_info = room_members.get(uid, {}) + profile_info = room_members.get(uid) if profile_info: display_name = profile_info.display_name else: From acb58bfb6a3523c67b3d426bb6a25b7329a4604f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Apr 2017 15:39:19 +0100 Subject: [PATCH 6/8] fix up --- synapse/push/bulk_push_rule_evaluator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index e40f62de78..f943ff640f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -91,7 +91,8 @@ class BulkPushRuleEvaluator: profile_info = room_members.get(uid) if profile_info: display_name = profile_info.display_name - else: + + if not display_name: # Handle the case where we are pushing a membership event to # that user, as they might not be already joined. if event.type == EventTypes.Member and event.state_key == uid: From f053a1409e2c0105859fe6c9af02efd49206eb91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 25 Apr 2017 17:22:55 +0100 Subject: [PATCH 7/8] Make state caches cache in ascii --- synapse/storage/state.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index e89001d994..a16afa8df5 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -16,6 +16,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches import intern_string +from synapse.util.stringutils import to_ascii from synapse.storage.engines import PostgresEngine from twisted.internet import defer @@ -89,7 +90,7 @@ class StateStore(SQLBaseStore): ) return { - (r[0], r[1]): r[2] for r in txn + (intern_string(r[0]), intern_string(r[1])): to_ascii(r[2]) for r in txn } return self.runInteraction( @@ -655,7 +656,7 @@ class StateStore(SQLBaseStore): state_dict = results[group] state_dict.update( - ((intern_string(k[0]), intern_string(k[1])), v) + ((intern_string(k[0]), intern_string(k[1])), to_ascii(v)) for k, v in group_state_dict.iteritems() ) From 94f36b027345a4a46b82e9dadfda084e0d9622fb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 25 Apr 2017 18:37:12 +0100 Subject: [PATCH 8/8] document how to make IPv6 work (#2088) * document how to make IPv6 work * spell out that pip will install 17.1 by default --- README.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.rst b/README.rst index 5be9eabdf3..d090ad86a2 100644 --- a/README.rst +++ b/README.rst @@ -245,6 +245,25 @@ Setting up a TURN server For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See ``_ for details. +IPv6 +---- + +As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph +for providing PR #1696. + +However, for federation to work on hosts with IPv6 DNS servers you **must** +be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002 +for details. We can't make Synapse depend on Twisted 17.1 by default +yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909) +so if you are using operating system dependencies you'll have to install your +own Twisted 17.1 package via pip or backports etc. + +If you're running in a virtualenv then pip should have installed the newest +Twisted automatically, but if your virtualenv is old you will need to manually +upgrade to a newer Twisted dependency via: + + pip install Twisted>=17.1.0 + Running Synapse ===============