bridges: Bring inter-realm (zulip) bridge into zulip/ & rename others.

zulip/integrations/bridge_with_matrix/README.md

@@ -0,0 +1,58 @@
+# Matrix <--> Zulip bridge
+
+This acts as a bridge between Matrix and Zulip. It also enables a
+Zulip topic to be federated between two Zulip servers.
+
+## Usage
+
+### For IRC bridges
+
+Matrix has been bridged to the listed
+[IRC networks](https://github.com/matrix-org/matrix-appservice-irc/wiki/Bridged-IRC-networks),
+where the 'Room alias format' refers to the `room_id` for the corresponding IRC channel.
+
+For example, for the freenode channel `#zulip-test`, the `room_id` would be
+`#freenode_#zulip-test:matrix.org`.
+
+Hence, this can also be used as a IRC <--> Zulip bridge.
+
+## Steps to configure the Matrix bridge
+
+To obtain a configuration file template, run the script with the
+`--write-sample-config` option to obtain a configuration file to fill in the
+details mentioned below. For example:
+
+* If you installed the `zulip` package: `zulip-matrix-bridge --write-sample-config matrix_bridge.conf`
+
+* If you are running from the Zulip GitHub repo: `python matrix_bridge.py --write-sample-config matrix_bridge.conf`
+
+### 1. Zulip endpoint
+1. Create a generic Zulip bot, with a full name like `IRC Bot` or `Matrix Bot`.
+2. Subscribe the bot user to the stream you'd like to bridge your IRC or Matrix
+   channel into.
+3. In the `zulip` section of the configuration file, enter the bot's `zuliprc`
+   details (`email`, `api_key`, and `site`).
+4. In the same section, also enter the Zulip `stream` and `topic`.
+
+### 2. Matrix endpoint
+1. Create a user on [matrix.org](https://matrix.org/), preferably with
+   a formal name like to `zulip-bot`.
+2. In the `matrix` section of the configuration file, enter the user's username
+   and password.
+3. Also enter the `host` and `room_id` into the same section.
+
+## Running the bridge
+
+After the steps above have been completed, assuming you have the configuration
+in a file called `matrix_bridge.conf`:
+
+* If you installed the `zulip` package: run `zulip-matrix-bridge -c matrix_bridge.conf`
+
+* If you are running from the Zulip GitHub repo: run `python matrix_bridge.py -c matrix_bridge.conf`
+
+## Caveats for IRC mirroring
+
+There are certain
+[IRC channels](https://github.com/matrix-org/matrix-appservice-irc/wiki/Channels-from-which-the-IRC-bridge-is-banned)
+where the Matrix.org IRC bridge has been banned for technical reasons.
+You can't mirror those IRC channels using this integration.

zulip/integrations/bridge_with_matrix/matrix_bridge.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python
+import os
+import logging
+import signal
+import traceback
+import zulip
+import sys
+import argparse
+import re
+import configparser
+
+from collections import OrderedDict
+
+from types import FrameType
+from typing import Any, Callable, Dict, Optional
+
+from matrix_client.errors import MatrixRequestError
+from matrix_client.client import MatrixClient
+from requests.exceptions import MissingSchema
+
+GENERAL_NETWORK_USERNAME_REGEX = '@_?[a-zA-Z0-9]+_([a-zA-Z0-9-_]+):[a-zA-Z0-9.]+'
+MATRIX_USERNAME_REGEX = '@([a-zA-Z0-9-_]+):matrix.org'
+
+# change these templates to change the format of displayed message
+ZULIP_MESSAGE_TEMPLATE = "**{username}**: {message}"
+MATRIX_MESSAGE_TEMPLATE = "<{username}> {message}"
+
+class Bridge_ConfigException(Exception):
+    pass
+
+class Bridge_FatalMatrixException(Exception):
+    pass
+
+class Bridge_ZulipFatalException(Exception):
+    pass
+
+def matrix_login(matrix_client, matrix_config):
+    # type: (Any, Dict[str, Any]) -> None
+    try:
+        matrix_client.login_with_password(matrix_config["username"],
+                                          matrix_config["password"])
+    except MatrixRequestError as exception:
+        if exception.code == 403:
+            raise Bridge_FatalMatrixException("Bad username or password.")
+        else:
+            raise Bridge_FatalMatrixException("Check if your server details are correct.")
+    except MissingSchema as exception:
+        raise Bridge_FatalMatrixException("Bad URL format.")
+
+def matrix_join_room(matrix_client, matrix_config):
+    # type: (Any, Dict[str, Any]) -> Any
+    try:
+        room = matrix_client.join_room(matrix_config["room_id"])
+        return room
+    except MatrixRequestError as exception:
+        if exception.code == 403:
+            raise Bridge_FatalMatrixException("Room ID/Alias in the wrong format")
+        else:
+            raise Bridge_FatalMatrixException("Couldn't find room.")
+
+def die(signal, frame):
+    # type: (int, FrameType) -> None
+    # We actually want to exit, so run os._exit (so as not to be caught and restarted)
+    os._exit(1)
+
+def matrix_to_zulip(zulip_client, zulip_config, matrix_config, no_noise):
+    # type: (zulip.Client, Dict[str, Any], Dict[str, Any], bool) -> Callable[[Any, Dict[str, Any]], None]
+    def _matrix_to_zulip(room, event):
+        # type: (Any, Dict[str, Any]) -> None
+        """
+        Matrix -> Zulip
+        """
+        content = get_message_content_from_event(event, no_noise)
+
+        zulip_bot_user = ('@%s:matrix.org' % matrix_config['username'])
+        # We do this to identify the messages generated from Zulip -> Matrix
+        # and we make sure we don't forward it again to the Zulip stream.
+        not_from_zulip_bot = ('body' not in event['content'] or
+                              event['sender'] != zulip_bot_user)
+
+        if not_from_zulip_bot and content:
+            try:
+                result = zulip_client.send_message({
+                    "sender": zulip_client.email,
+                    "type": "stream",
+                    "to": zulip_config["stream"],
+                    "subject": zulip_config["topic"],
+                    "content": content,
+                })
+            except Exception as exception:  # XXX This should be more specific
+                # Generally raised when user is forbidden
+                raise Bridge_ZulipFatalException(exception)
+            if result['result'] != 'success':
+                # Generally raised when API key is invalid
+                raise Bridge_ZulipFatalException(result['msg'])
+
+    return _matrix_to_zulip
+
+def get_message_content_from_event(event, no_noise):
+    # type: (Dict[str, Any], bool) -> Optional[str]
+    irc_nick = shorten_irc_nick(event['sender'])
+    if event['type'] == "m.room.member":
+        if no_noise:
+            return None
+        # Join and leave events can be noisy. They are ignored by default.
+        # To enable these events pass `no_noise` as `False` as the script argument
+        if event['membership'] == "join":
+            content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick,
+                                                    message="joined")
+        elif event['membership'] == "leave":
+            content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick,
+                                                    message="quit")
+    elif event['type'] == "m.room.message":
+        if event['content']['msgtype'] == "m.text" or event['content']['msgtype'] == "m.emote":
+            content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick,
+                                                    message=event['content']['body'])
+    else:
+        content = event['type']
+    return content
+
+def shorten_irc_nick(nick):
+    # type: (str) -> str
+    """
+    Add nick shortner functions for specific IRC networks
+    Eg: For freenode change '@freenode_user:matrix.org' to 'user'
+    Check the list of IRC networks here:
+    https://github.com/matrix-org/matrix-appservice-irc/wiki/Bridged-IRC-networks
+    """
+    match = re.match(GENERAL_NETWORK_USERNAME_REGEX, nick)
+    if match:
+        return match.group(1)
+    # For matrix users
+    match = re.match(MATRIX_USERNAME_REGEX, nick)
+    if match:
+        return match.group(1)
+    return nick
+
+def zulip_to_matrix(config, room):
+    # type: (Dict[str, Any], Any) -> Callable[[Dict[str, Any]], None]
+
+    def _zulip_to_matrix(msg):
+        # type: (Dict[str, Any]) -> None
+        """
+        Zulip -> Matrix
+        """
+        message_valid = check_zulip_message_validity(msg, config)
+        if message_valid:
+            matrix_username = msg["sender_full_name"].replace(' ', '')
+            matrix_text = MATRIX_MESSAGE_TEMPLATE.format(username=matrix_username,
+                                                         message=msg["content"])
+            # Forward Zulip message to Matrix
+            room.send_text(matrix_text)
+    return _zulip_to_matrix
+
+def check_zulip_message_validity(msg, config):
+    # type: (Dict[str, Any], Dict[str, Any]) -> bool
+    is_a_stream = msg["type"] == "stream"
+    in_the_specified_stream = msg["display_recipient"] == config["stream"]
+    at_the_specified_subject = msg["subject"] == config["topic"]
+
+    # We do this to identify the messages generated from Matrix -> Zulip
+    # and we make sure we don't forward it again to the Matrix.
+    not_from_zulip_bot = msg["sender_email"] != config["email"]
+    if is_a_stream and not_from_zulip_bot and in_the_specified_stream and at_the_specified_subject:
+        return True
+    return False
+
+def generate_parser():
+    # type: () -> argparse.ArgumentParser
+    description = """
+    Script to bridge between a topic in a Zulip stream, and a Matrix channel.
+
+    Tested connections:
+        * Zulip <-> Matrix channel
+        * Zulip <-> IRC channel (bridged via Matrix)
+
+    Example matrix 'room_id' options might be, if via matrix.org:
+        * #zulip:matrix.org (zulip channel on Matrix)
+        * #freenode_#zulip:matrix.org (zulip channel on irc.freenode.net)"""
+
+    parser = argparse.ArgumentParser(description=description,
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument('-c', '--config', required=False,
+                        help="Path to the config file for the bridge.")
+    parser.add_argument('--write-sample-config', metavar='PATH', dest='sample_config',
+                        help="Generate a configuration template at the specified location.")
+    parser.add_argument('--from-zuliprc', metavar='ZULIPRC', dest='zuliprc',
+                        help="Optional path to zuliprc file for bot, when using --write-sample-config")
+    parser.add_argument('--show-join-leave', dest='no_noise',
+                        default=True, action='store_false',
+                        help="Enable IRC join/leave events.")
+    return parser
+
+def read_configuration(config_file):
+    # type: (str) -> Dict[str, Dict[str, str]]
+    config = configparser.ConfigParser()
+
+    try:
+        config.read(config_file)
+    except configparser.Error as exception:
+        raise Bridge_ConfigException(str(exception))
+
+    if set(config.sections()) != {'matrix', 'zulip'}:
+        raise Bridge_ConfigException("Please ensure the configuration has zulip & matrix sections.")
+
+    # TODO Could add more checks for configuration content here
+
+    return {section: dict(config[section]) for section in config.sections()}
+
+def write_sample_config(target_path, zuliprc):
+    # type: (str, Optional[str]) -> None
+    if os.path.exists(target_path):
+        raise Bridge_ConfigException("Path '{}' exists; not overwriting existing file.".format(target_path))
+
+    sample_dict = OrderedDict((
+        ('matrix', OrderedDict((
+            ('host', 'https://matrix.org'),
+            ('username', 'username'),
+            ('password', 'password'),
+            ('room_id', '#zulip:matrix.org'),
+        ))),
+        ('zulip', OrderedDict((
+            ('email', 'glitch-bot@chat.zulip.org'),
+            ('api_key', 'aPiKeY'),
+            ('site', 'https://chat.zulip.org'),
+            ('stream', 'test here'),
+            ('topic', 'matrix'),
+        ))),
+    ))
+
+    if zuliprc is not None:
+        if not os.path.exists(zuliprc):
+            raise Bridge_ConfigException("Zuliprc file '{}' does not exist.".format(zuliprc))
+
+        zuliprc_config = configparser.ConfigParser()
+        try:
+            zuliprc_config.read(zuliprc)
+        except configparser.Error as exception:
+            raise Bridge_ConfigException(str(exception))
+
+        # Can add more checks for validity of zuliprc file here
+
+        sample_dict['zulip']['email'] = zuliprc_config['api']['email']
+        sample_dict['zulip']['site'] = zuliprc_config['api']['site']
+        sample_dict['zulip']['api_key'] = zuliprc_config['api']['key']
+
+    sample = configparser.ConfigParser()
+    sample.read_dict(sample_dict)
+    with open(target_path, 'w') as target:
+        sample.write(target)
+
+def main():
+    # type: () -> None
+    signal.signal(signal.SIGINT, die)
+    logging.basicConfig(level=logging.WARNING)
+
+    parser = generate_parser()
+    options = parser.parse_args()
+
+    if options.sample_config:
+        try:
+            write_sample_config(options.sample_config, options.zuliprc)
+        except Bridge_ConfigException as exception:
+            sys.exit(exception)
+        if options.zuliprc is None:
+            print("Wrote sample configuration to '{}'".format(options.sample_config))
+        else:
+            print("Wrote sample configuration to '{}' using zuliprc file '{}'"
+                  .format(options.sample_config, options.zuliprc))
+        sys.exit(0)
+    elif not options.config:
+        print("Options required: -c or --config to run, OR --write-sample-config.")
+        parser.print_usage()
+        sys.exit(1)
+
+    try:
+        config = read_configuration(options.config)
+    except Bridge_ConfigException as exception:
+        sys.exit("Could not parse config file: {}".format(exception))
+
+    # Get config for each client
+    zulip_config = config["zulip"]
+    matrix_config = config["matrix"]
+
+    # Initiate clients
+    backoff = zulip.RandomExponentialBackoff(timeout_success_equivalent=300)
+    while backoff.keep_going():
+        print("Starting matrix mirroring bot")
+        try:
+            zulip_client = zulip.Client(email=zulip_config["email"],
+                                        api_key=zulip_config["api_key"],
+                                        site=zulip_config["site"])
+            matrix_client = MatrixClient(matrix_config["host"])
+
+            # Login to Matrix
+            matrix_login(matrix_client, matrix_config)
+            # Join a room in Matrix
+            room = matrix_join_room(matrix_client, matrix_config)
+
+            room.add_listener(matrix_to_zulip(zulip_client, zulip_config, matrix_config,
+                                              options.no_noise))
+
+            print("Starting listener thread on Matrix client")
+            matrix_client.start_listener_thread()
+
+            print("Starting message handler on Zulip client")
+            zulip_client.call_on_each_message(zulip_to_matrix(zulip_config, room))
+
+        except Bridge_FatalMatrixException as exception:
+            sys.exit("Matrix bridge error: {}".format(exception))
+        except Bridge_ZulipFatalException as exception:
+            sys.exit("Zulip bridge error: {}".format(exception))
+        except zulip.ZulipError as exception:
+            sys.exit("Zulip error: {}".format(exception))
+        except Exception as e:
+            traceback.print_exc()
+        backoff.fail()
+
+if __name__ == '__main__':
+    main()

zulip/integrations/bridge_with_matrix/requirements.txt

@@ -0,0 +1 @@
+matrix-client==0.2.0

zulip/integrations/bridge_with_matrix/test_matrix.py

@@ -0,0 +1,143 @@
+from matrix_bridge import (
+    check_zulip_message_validity,
+    zulip_to_matrix,
+)
+
+from unittest import TestCase, mock
+from subprocess import Popen, PIPE
+import os
+
+import shutil
+
+from contextlib import contextmanager
+from tempfile import mkdtemp
+
+script_file = "matrix_bridge.py"
+script_dir = os.path.dirname(__file__)
+script = os.path.join(script_dir, script_file)
+
+from typing import List, Iterator
+
+sample_config_path = "matrix_test.conf"
+
+sample_config_text = """[matrix]
+host = https://matrix.org
+username = username
+password = password
+room_id = #zulip:matrix.org
+
+[zulip]
+email = glitch-bot@chat.zulip.org
+api_key = aPiKeY
+site = https://chat.zulip.org
+stream = test here
+topic = matrix
+
+"""
+
+@contextmanager
+def new_temp_dir():
+    # type: () -> Iterator[str]
+    path = mkdtemp()
+    yield path
+    shutil.rmtree(path)
+
+class MatrixBridgeScriptTests(TestCase):
+    def output_from_script(self, options):
+        # type: (List[str]) -> List[str]
+        popen = Popen(["python", script] + options, stdin=PIPE, stdout=PIPE, universal_newlines=True)
+        return popen.communicate()[0].strip().split("\n")
+
+    def test_no_args(self):
+        # type: () -> None
+        output_lines = self.output_from_script([])
+        expected_lines = [
+            "Options required: -c or --config to run, OR --write-sample-config.",
+            "usage: {} [-h]".format(script_file)
+        ]
+        for expected, output in zip(expected_lines, output_lines):
+            self.assertIn(expected, output)
+
+    def test_help_usage_and_description(self):
+        # type: () -> None
+        output_lines = self.output_from_script(["-h"])
+        usage = "usage: {} [-h]".format(script_file)
+        description = "Script to bridge"
+        self.assertIn(usage, output_lines[0])
+        blank_lines = [num for num, line in enumerate(output_lines) if line == '']
+        # There should be blank lines in the output
+        self.assertTrue(blank_lines)
+        # There should be finite output
+        self.assertTrue(len(output_lines) > blank_lines[0])
+        # Minimal description should be in the first line of the 2nd "paragraph"
+        self.assertIn(description, output_lines[blank_lines[0] + 1])
+
+    def test_write_sample_config(self):
+        # type: () -> None
+        with new_temp_dir() as tempdir:
+            path = os.path.join(tempdir, sample_config_path)
+            output_lines = self.output_from_script(["--write-sample-config", path])
+            self.assertEqual(output_lines, ["Wrote sample configuration to '{}'".format(path)])
+
+            with open(path) as sample_file:
+                self.assertEqual(sample_file.read(), sample_config_text)
+
+
+class MatrixBridgeZulipToMatrixTests(TestCase):
+    valid_zulip_config = dict(
+        stream="some stream",
+        topic="some topic",
+        email="some@email"
+    )
+    valid_msg = dict(
+        sender_email="John@Smith.smith",  # must not be equal to config:email
+        type="stream",  # Can only mirror Zulip streams
+        display_recipient=valid_zulip_config['stream'],
+        subject=valid_zulip_config['topic']
+    )
+
+    def test_zulip_message_validity_success(self):
+        # type: () -> None
+        zulip_config = self.valid_zulip_config
+        msg = self.valid_msg
+        # Ensure the test inputs are valid for success
+        assert msg['sender_email'] != zulip_config['email']
+
+        self.assertTrue(check_zulip_message_validity(msg, zulip_config))
+
+    def test_zulip_message_validity_failure(self):
+        # type: () -> None
+        zulip_config = self.valid_zulip_config
+
+        msg_wrong_stream = dict(self.valid_msg, display_recipient='foo')
+        self.assertFalse(check_zulip_message_validity(msg_wrong_stream, zulip_config))
+
+        msg_wrong_topic = dict(self.valid_msg, subject='foo')
+        self.assertFalse(check_zulip_message_validity(msg_wrong_topic, zulip_config))
+
+        msg_not_stream = dict(self.valid_msg, type="private")
+        self.assertFalse(check_zulip_message_validity(msg_not_stream, zulip_config))
+
+        msg_from_bot = dict(self.valid_msg, sender_email=zulip_config['email'])
+        self.assertFalse(check_zulip_message_validity(msg_from_bot, zulip_config))
+
+    def test_zulip_to_matrix(self):
+        # type: () -> None
+        room = mock.MagicMock()
+        zulip_config = self.valid_zulip_config
+        send_msg = zulip_to_matrix(zulip_config, room)
+
+        msg = dict(self.valid_msg, sender_full_name="John Smith")
+
+        expected = {
+            'hi': '{} hi',
+            '*hi*': '{} *hi*',
+            '**hi**': '{} **hi**',
+        }
+
+        for content in expected:
+            send_msg(dict(msg, content=content))
+
+        for (method, params, _), expect in zip(room.method_calls, expected.values()):
+            self.assertEqual(method, 'send_text')
+            self.assertEqual(params[0], expect.format('<JohnSmith>'))