xenofem/python-zulip-api
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

black: Reformat skipping string normalization.

Browse source
This commit is contained in:
PIG208 2021-05-28 17:03:46 +08:00 • committed by Tim Abbott
parent 5580c68ae5
commit fba21bb00d
178 changed files with 6562 additions and 4469 deletions
Download patch file Download diff file Expand all files Collapse all files

6
zulip/integrations/bridge_between_zulips/interrealm_bridge_config.py
View file

@ -4,11 +4,13 @@ config = {
"api_key": "key1",
"site": "https://realm1.zulipchat.com",
"stream": "bridges",
"subject": "<- realm2"},
"subject": "<- realm2",
},
"bot_2": {
"email": "tunnel-bot@realm2.zulipchat.com",
"api_key": "key2",
"site": "https://realm2.zulipchat.com",
"stream": "bridges",
"subject": "<- realm1"}
"subject": "<- realm1",
},
}

26
zulip/integrations/bridge_between_zulips/run-interrealm-bridge
View file

@ -11,9 +11,9 @@ import interrealm_bridge_config
import zulip
def create_pipe_event(to_client: zulip.Client, from_bot: Dict[str, Any],
to_bot: Dict[str, Any], stream_wide: bool
) -> Callable[[Dict[str, Any]], None]:
def create_pipe_event(
to_client: zulip.Client, from_bot: Dict[str, Any], to_bot: Dict[str, Any], stream_wide: bool
) -> Callable[[Dict[str, Any]], None]:
def _pipe_message(msg: Dict[str, Any]) -> None:
isa_stream = msg["type"] == "stream"
not_from_bot = msg["sender_email"] not in (from_bot["email"], to_bot["email"])
@ -32,8 +32,9 @@ def create_pipe_event(to_client: zulip.Client, from_bot: Dict[str, Any],
if "/user_uploads/" in msg["content"]:
# Fix the upload URL of the image to be the source of where it
# comes from
msg["content"] = msg["content"].replace("/user_uploads/",
from_bot["site"] + "/user_uploads/")
msg["content"] = msg["content"].replace(
"/user_uploads/", from_bot["site"] + "/user_uploads/"
)
if msg["content"].startswith(("```", "- ", "* ", "> ", "1. ")):
# If a message starts with special prefixes, make sure to prepend a newline for
# formatting purpose
@ -45,7 +46,7 @@ def create_pipe_event(to_client: zulip.Client, from_bot: Dict[str, Any],
"content": "**{}**: {}".format(msg["sender_full_name"], msg["content"]),
"has_attachment": msg.get("has_attachment", False),
"has_image": msg.get("has_image", False),
"has_link": msg.get("has_link", False)
"has_link": msg.get("has_link", False),
}
print(msg_data)
print(to_client.send_message(msg_data))
@ -55,8 +56,10 @@ def create_pipe_event(to_client: zulip.Client, from_bot: Dict[str, Any],
if event["type"] == "message":
msg = event["message"]
_pipe_message(msg)
return _pipe_event
if __name__ == "__main__":
usage = """run-interrealm-bridge [--stream]
@ -71,20 +74,15 @@ if __name__ == "__main__":
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument('--stream',
action='store_true',
help="",
default=False)
parser.add_argument('--stream', action='store_true', help="", default=False)
args = parser.parse_args()
options = interrealm_bridge_config.config
bot1 = options["bot_1"]
bot2 = options["bot_2"]
client1 = zulip.Client(email=bot1["email"], api_key=bot1["api_key"],
site=bot1["site"])
client2 = zulip.Client(email=bot2["email"], api_key=bot2["api_key"],
site=bot2["site"])
client1 = zulip.Client(email=bot1["email"], api_key=bot1["api_key"], site=bot1["site"])
client2 = zulip.Client(email=bot2["email"], api_key=bot2["api_key"], site=bot2["site"])
# A bidirectional tunnel
pipe_event1 = create_pipe_event(client2, bot1, bot2, args.stream)
p1 = mp.Process(target=client1.call_on_each_event, args=(pipe_event1, ["message"]))

22
zulip/integrations/bridge_with_irc/irc-mirror.py
View file

@ -26,7 +26,9 @@ Note that "_zulip" will be automatically appended to the IRC nick provided
"""
if __name__ == "__main__":
parser = zulip.add_default_arguments(argparse.ArgumentParser(usage=usage), allow_provisioning=True)
parser = zulip.add_default_arguments(
argparse.ArgumentParser(usage=usage), allow_provisioning=True
)
parser.add_argument('--irc-server', default=None)
parser.add_argument('--port', default=6667)
parser.add_argument('--nick-prefix', default=None)
@ -43,14 +45,24 @@ if __name__ == "__main__":
from irc_mirror_backend import IRCBot
except ImportError:
traceback.print_exc()
print("You have unsatisfied dependencies. Install all missing dependencies with "
"{} --provision".format(sys.argv[0]))
print(
"You have unsatisfied dependencies. Install all missing dependencies with "
"{} --provision".format(sys.argv[0])
)
sys.exit(1)
if options.irc_server is None or options.nick_prefix is None or options.channel is None:
parser.error("Missing required argument")
nickname = options.nick_prefix + "_zulip"
bot = IRCBot(zulip_client, options.stream, options.topic, options.channel,
nickname, options.irc_server, options.nickserv_pw, options.port)
bot = IRCBot(
zulip_client,
options.stream,
options.topic,
options.channel,
nickname,
options.irc_server,
options.nickserv_pw,
options.port,
)
bot.start()

61
zulip/integrations/bridge_with_irc/irc_mirror_backend.py
View file

@ -10,8 +10,17 @@ from irc.client_aio import AioReactor
class IRCBot(irc.bot.SingleServerIRCBot):
reactor_class = AioReactor
def __init__(self, zulip_client: Any, stream: str, topic: str, channel: irc.bot.Channel,
nickname: str, server: str, nickserv_password: str = '', port: int = 6667) -> None:
def __init__(
self,
zulip_client: Any,
stream: str,
topic: str,
channel: irc.bot.Channel,
nickname: str,
server: str,
nickserv_password: str = '',
port: int = 6667,
) -> None:
self.channel = channel # type: irc.bot.Channel
self.zulip_client = zulip_client
self.stream = stream
@ -31,9 +40,7 @@ class IRCBot(irc.bot.SingleServerIRCBot):
# Taken from
# https://github.com/jaraco/irc/blob/master/irc/client_aio.py,
# in particular the method of AioSimpleIRCClient
self.c = self.reactor.loop.run_until_complete(
self.connection.connect(*args, **kwargs)
)
self.c = self.reactor.loop.run_until_complete(self.connection.connect(*args, **kwargs))
print("Listening now. Please send an IRC message to verify operation")
def check_subscription_or_die(self) -> None:
@ -43,7 +50,10 @@ class IRCBot(irc.bot.SingleServerIRCBot):
exit(1)
subs = [s["name"] for s in resp["subscriptions"]]
if self.stream not in subs:
print("The bot is not yet subscribed to stream '%s'. Please subscribe the bot to the stream first." % (self.stream,))
print(
"The bot is not yet subscribed to stream '%s'. Please subscribe the bot to the stream first."
% (self.stream,)
)
exit(1)
def on_nicknameinuse(self, c: ServerConnection, e: Event) -> None:
@ -70,8 +80,11 @@ class IRCBot(irc.bot.SingleServerIRCBot):
else:
return
else:
recipients = [u["short_name"] for u in msg["display_recipient"] if
u["email"] != msg["sender_email"]]
recipients = [
u["short_name"]
for u in msg["display_recipient"]
if u["email"] != msg["sender_email"]
]
if len(recipients) == 1:
send = lambda x: self.c.privmsg(recipients[0], x)
else:
@ -89,12 +102,16 @@ class IRCBot(irc.bot.SingleServerIRCBot):
return
# Forward the PM to Zulip
print(self.zulip_client.send_message({
"sender": sender,
"type": "private",
"to": "username@example.com",
"content": content,
}))
print(
self.zulip_client.send_message(
{
"sender": sender,
"type": "private",
"to": "username@example.com",
"content": content,
}
)
)
def on_pubmsg(self, c: ServerConnection, e: Event) -> None:
content = e.arguments[0]
@ -103,12 +120,16 @@ class IRCBot(irc.bot.SingleServerIRCBot):
return
# Forward the stream message to Zulip
print(self.zulip_client.send_message({
"type": "stream",
"to": self.stream,
"subject": self.topic,
"content": "**{}**: {}".format(sender, content),
}))
print(
self.zulip_client.send_message(
{
"type": "stream",
"to": self.stream,
"subject": self.topic,
"content": "**{}**: {}".format(sender, content),
}
)
)
def on_dccmsg(self, c: ServerConnection, e: Event) -> None:
c.privmsg("You said: " + e.arguments[0])

153
zulip/integrations/bridge_with_matrix/matrix_bridge.py
View file

@ -24,19 +24,22 @@ MATRIX_USERNAME_REGEX = '@([a-zA-Z0-9-_]+):matrix.org'
ZULIP_MESSAGE_TEMPLATE = "**{username}**: {message}"
MATRIX_MESSAGE_TEMPLATE = "<{username}> {message}"
class Bridge_ConfigException(Exception):
pass
class Bridge_FatalMatrixException(Exception):
pass
class Bridge_ZulipFatalException(Exception):
pass
def matrix_login(matrix_client: Any, matrix_config: Dict[str, Any]) -> None:
try:
matrix_client.login_with_password(matrix_config["username"],
matrix_config["password"])
matrix_client.login_with_password(matrix_config["username"], matrix_config["password"])
except MatrixRequestError as exception:
if exception.code == 403:
raise Bridge_FatalMatrixException("Bad username or password.")
@ -45,6 +48,7 @@ def matrix_login(matrix_client: Any, matrix_config: Dict[str, Any]) -> None:
except MissingSchema:
raise Bridge_FatalMatrixException("Bad URL format.")
def matrix_join_room(matrix_client: Any, matrix_config: Dict[str, Any]) -> Any:
try:
room = matrix_client.join_room(matrix_config["room_id"])
@ -55,10 +59,12 @@ def matrix_join_room(matrix_client: Any, matrix_config: Dict[str, Any]) -> Any:
else:
raise Bridge_FatalMatrixException("Couldn't find room.")
def die(signal: int, frame: FrameType) -> None:
# We actually want to exit, so run os._exit (so as not to be caught and restarted)
os._exit(1)
def matrix_to_zulip(
zulip_client: zulip.Client,
zulip_config: Dict[str, Any],
@ -78,12 +84,14 @@ def matrix_to_zulip(
if not_from_zulip_bot and content:
try:
result = zulip_client.send_message({
"type": "stream",
"to": zulip_config["stream"],
"subject": zulip_config["topic"],
"content": content,
})
result = zulip_client.send_message(
{
"type": "stream",
"to": zulip_config["stream"],
"subject": zulip_config["topic"],
"content": content,
}
)
except Exception as exception: # XXX This should be more specific
# Generally raised when user is forbidden
raise Bridge_ZulipFatalException(exception)
@ -93,6 +101,7 @@ def matrix_to_zulip(
return _matrix_to_zulip
def get_message_content_from_event(event: Dict[str, Any], no_noise: bool) -> Optional[str]:
irc_nick = shorten_irc_nick(event['sender'])
if event['type'] == "m.room.member":
@ -101,19 +110,19 @@ def get_message_content_from_event(event: Dict[str, Any], no_noise: bool) -> Opt
# Join and leave events can be noisy. They are ignored by default.
# To enable these events pass `no_noise` as `False` as the script argument
if event['membership'] == "join":
content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick,
message="joined")
content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick, message="joined")
elif event['membership'] == "leave":
content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick,
message="quit")
content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick, message="quit")
elif event['type'] == "m.room.message":
if event['content']['msgtype'] == "m.text" or event['content']['msgtype'] == "m.emote":
content = ZULIP_MESSAGE_TEMPLATE.format(username=irc_nick,
message=event['content']['body'])
content = ZULIP_MESSAGE_TEMPLATE.format(
username=irc_nick, message=event['content']['body']
)
else:
content = event['type']
return content
def shorten_irc_nick(nick: str) -> str:
"""
Add nick shortner functions for specific IRC networks
@ -130,8 +139,8 @@ def shorten_irc_nick(nick: str) -> str:
return match.group(1)
return nick
def zulip_to_matrix(config: Dict[str, Any], room: Any) -> Callable[[Dict[str, Any]], None]:
def zulip_to_matrix(config: Dict[str, Any], room: Any) -> Callable[[Dict[str, Any]], None]:
def _zulip_to_matrix(msg: Dict[str, Any]) -> None:
"""
Zulip -> Matrix
@ -139,12 +148,15 @@ def zulip_to_matrix(config: Dict[str, Any], room: Any) -> Callable[[Dict[str, An
message_valid = check_zulip_message_validity(msg, config)
if message_valid:
matrix_username = msg["sender_full_name"].replace(' ', '')
matrix_text = MATRIX_MESSAGE_TEMPLATE.format(username=matrix_username,
message=msg["content"])
matrix_text = MATRIX_MESSAGE_TEMPLATE.format(
username=matrix_username, message=msg["content"]
)
# Forward Zulip message to Matrix
room.send_text(matrix_text)
return _zulip_to_matrix
def check_zulip_message_validity(msg: Dict[str, Any], config: Dict[str, Any]) -> bool:
is_a_stream = msg["type"] == "stream"
in_the_specified_stream = msg["display_recipient"] == config["stream"]
@ -157,6 +169,7 @@ def check_zulip_message_validity(msg: Dict[str, Any], config: Dict[str, Any]) ->
return True
return False
def generate_parser() -> argparse.ArgumentParser:
description = """
Script to bridge between a topic in a Zulip stream, and a Matrix channel.
@ -169,19 +182,34 @@ def generate_parser() -> argparse.ArgumentParser:
* #zulip:matrix.org (zulip channel on Matrix)
* #freenode_#zulip:matrix.org (zulip channel on irc.freenode.net)"""
parser = argparse.ArgumentParser(description=description,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-c', '--config', required=False,
help="Path to the config file for the bridge.")
parser.add_argument('--write-sample-config', metavar='PATH', dest='sample_config',
help="Generate a configuration template at the specified location.")
parser.add_argument('--from-zuliprc', metavar='ZULIPRC', dest='zuliprc',
help="Optional path to zuliprc file for bot, when using --write-sample-config")
parser.add_argument('--show-join-leave', dest='no_noise',
default=True, action='store_false',
help="Enable IRC join/leave events.")
parser = argparse.ArgumentParser(
description=description, formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument(
'-c', '--config', required=False, help="Path to the config file for the bridge."
)
parser.add_argument(
'--write-sample-config',
metavar='PATH',
dest='sample_config',
help="Generate a configuration template at the specified location.",
)
parser.add_argument(
'--from-zuliprc',
metavar='ZULIPRC',
dest='zuliprc',
help="Optional path to zuliprc file for bot, when using --write-sample-config",
)
parser.add_argument(
'--show-join-leave',
dest='no_noise',
default=True,
action='store_false',
help="Enable IRC join/leave events.",
)
return parser
def read_configuration(config_file: str) -> Dict[str, Dict[str, str]]:
config = configparser.ConfigParser()
@ -197,25 +225,40 @@ def read_configuration(config_file: str) -> Dict[str, Dict[str, str]]:
return {section: dict(config[section]) for section in config.sections()}
def write_sample_config(target_path: str, zuliprc: Optional[str]) -> None:
if os.path.exists(target_path):
raise Bridge_ConfigException("Path '{}' exists; not overwriting existing file.".format(target_path))
raise Bridge_ConfigException(
"Path '{}' exists; not overwriting existing file.".format(target_path)
)
sample_dict = OrderedDict((
('matrix', OrderedDict((
('host', 'https://matrix.org'),
('username', 'username'),
('password', 'password'),
('room_id', '#zulip:matrix.org'),
))),
('zulip', OrderedDict((
('email', 'glitch-bot@chat.zulip.org'),
('api_key', 'aPiKeY'),
('site', 'https://chat.zulip.org'),
('stream', 'test here'),
('topic', 'matrix'),
))),
))
sample_dict = OrderedDict(
(
(
'matrix',
OrderedDict(
(
('host', 'https://matrix.org'),
('username', 'username'),
('password', 'password'),
('room_id', '#zulip:matrix.org'),
)
),
),
(
'zulip',
OrderedDict(
(
('email', 'glitch-bot@chat.zulip.org'),
('api_key', 'aPiKeY'),
('site', 'https://chat.zulip.org'),
('stream', 'test here'),
('topic', 'matrix'),
)
),
),
)
)
if zuliprc is not None:
if not os.path.exists(zuliprc):
@ -238,6 +281,7 @@ def write_sample_config(target_path: str, zuliprc: Optional[str]) -> None:
with open(target_path, 'w') as target:
sample.write(target)
def main() -> None:
signal.signal(signal.SIGINT, die)
logging.basicConfig(level=logging.WARNING)
@ -254,8 +298,11 @@ def main() -> None:
if options.zuliprc is None:
print("Wrote sample configuration to '{}'".format(options.sample_config))
else:
print("Wrote sample configuration to '{}' using zuliprc file '{}'"
.format(options.sample_config, options.zuliprc))
print(
"Wrote sample configuration to '{}' using zuliprc file '{}'".format(
options.sample_config, options.zuliprc
)
)
sys.exit(0)
elif not options.config:
print("Options required: -c or --config to run, OR --write-sample-config.")
@ -277,9 +324,11 @@ def main() -> None:
while backoff.keep_going():
print("Starting matrix mirroring bot")
try:
zulip_client = zulip.Client(email=zulip_config["email"],
api_key=zulip_config["api_key"],
site=zulip_config["site"])
zulip_client = zulip.Client(
email=zulip_config["email"],
api_key=zulip_config["api_key"],
site=zulip_config["site"],
)
matrix_client = MatrixClient(matrix_config["host"])
# Login to Matrix
@ -287,8 +336,9 @@ def main() -> None:
# Join a room in Matrix
room = matrix_join_room(matrix_client, matrix_config)
room.add_listener(matrix_to_zulip(zulip_client, zulip_config, matrix_config,
options.no_noise))
room.add_listener(
matrix_to_zulip(zulip_client, zulip_config, matrix_config, options.no_noise)
)
print("Starting listener thread on Matrix client")
matrix_client.start_listener_thread()
@ -306,5 +356,6 @@ def main() -> None:
traceback.print_exc()
backoff.fail()
if __name__ == '__main__':
main()

57
zulip/integrations/bridge_with_matrix/test_matrix.py
View file

@ -30,22 +30,26 @@ topic = matrix
"""
@contextmanager
def new_temp_dir() -> Iterator[str]:
path = mkdtemp()
yield path
shutil.rmtree(path)
class MatrixBridgeScriptTests(TestCase):
def output_from_script(self, options: List[str]) -> List[str]:
popen = Popen(["python", script] + options, stdin=PIPE, stdout=PIPE, universal_newlines=True)
popen = Popen(
["python", script] + options, stdin=PIPE, stdout=PIPE, universal_newlines=True
)
return popen.communicate()[0].strip().split("\n")
def test_no_args(self) -> None:
output_lines = self.output_from_script([])
expected_lines = [
"Options required: -c or --config to run, OR --write-sample-config.",
"usage: {} [-h]".format(script_file)
"usage: {} [-h]".format(script_file),
]
for expected, output in zip(expected_lines, output_lines):
self.assertIn(expected, output)
@ -74,19 +78,27 @@ class MatrixBridgeScriptTests(TestCase):
def test_write_sample_config_from_zuliprc(self) -> None:
zuliprc_template = ["[api]", "email={email}", "key={key}", "site={site}"]
zulip_params = {'email': 'foo@bar',
'key': 'some_api_key',
'site': 'https://some.chat.serverplace'}
zulip_params = {
'email': 'foo@bar',
'key': 'some_api_key',
'site': 'https://some.chat.serverplace',
}
with new_temp_dir() as tempdir:
path = os.path.join(tempdir, sample_config_path)
zuliprc_path = os.path.join(tempdir, "zuliprc")
with open(zuliprc_path, "w") as zuliprc_file:
zuliprc_file.write("\n".join(zuliprc_template).format(**zulip_params))
output_lines = self.output_from_script(["--write-sample-config", path,
"--from-zuliprc", zuliprc_path])
self.assertEqual(output_lines,
["Wrote sample configuration to '{}' using zuliprc file '{}'"
.format(path, zuliprc_path)])
output_lines = self.output_from_script(
["--write-sample-config", path, "--from-zuliprc", zuliprc_path]
)
self.assertEqual(
output_lines,
[
"Wrote sample configuration to '{}' using zuliprc file '{}'".format(
path, zuliprc_path
)
],
)
with open(path) as sample_file:
sample_lines = [line.strip() for line in sample_file.readlines()]
@ -101,23 +113,26 @@ class MatrixBridgeScriptTests(TestCase):
path = os.path.join(tempdir, sample_config_path)
zuliprc_path = os.path.join(tempdir, "zuliprc")
# No writing of zuliprc file here -> triggers check for zuliprc absence
output_lines = self.output_from_script(["--write-sample-config", path,
"--from-zuliprc", zuliprc_path])
self.assertEqual(output_lines,
["Could not write sample config: Zuliprc file '{}' does not exist."
.format(zuliprc_path)])
output_lines = self.output_from_script(
["--write-sample-config", path, "--from-zuliprc", zuliprc_path]
)
self.assertEqual(
output_lines,
[
"Could not write sample config: Zuliprc file '{}' does not exist.".format(
zuliprc_path
)
],
)
class MatrixBridgeZulipToMatrixTests(TestCase):
valid_zulip_config = dict(
stream="some stream",
topic="some topic",
email="some@email"
)
valid_zulip_config = dict(stream="some stream", topic="some topic", email="some@email")
valid_msg = dict(
sender_email="John@Smith.smith", # must not be equal to config:email
type="stream", # Can only mirror Zulip streams
display_recipient=valid_zulip_config['stream'],
subject=valid_zulip_config['topic']
subject=valid_zulip_config['topic'],
)
def test_zulip_message_validity_success(self) -> None:

2
zulip/integrations/bridge_with_slack/bridge_with_slack_config.py
View file

@ -10,5 +10,5 @@ config = {
"username": "slack username",
"token": "slack token",
"channel": "C5Z5N7R8A -- must be channel id",
}
},
}

26
zulip/integrations/bridge_with_slack/run-slack-bridge
View file

@ -18,6 +18,7 @@ import zulip
ZULIP_MESSAGE_TEMPLATE = "**{username}**: {message}"
SLACK_MESSAGE_TEMPLATE = "<{username}> {message}"
def check_zulip_message_validity(msg: Dict[str, Any], config: Dict[str, Any]) -> bool:
is_a_stream = msg["type"] == "stream"
in_the_specified_stream = msg["display_recipient"] == config["stream"]
@ -30,6 +31,7 @@ def check_zulip_message_validity(msg: Dict[str, Any], config: Dict[str, Any]) ->
return True
return False
class SlackBridge:
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
@ -40,7 +42,8 @@ class SlackBridge:
self.zulip_client = zulip.Client(
email=self.zulip_config["email"],
api_key=self.zulip_config["api_key"],
site=self.zulip_config["site"])
site=self.zulip_config["site"],
)
self.zulip_stream = self.zulip_config["stream"]
self.zulip_subject = self.zulip_config["topic"]
@ -69,18 +72,22 @@ class SlackBridge:
message_valid = check_zulip_message_validity(msg, self.zulip_config)
if message_valid:
self.wrap_slack_mention_with_bracket(msg)
slack_text = SLACK_MESSAGE_TEMPLATE.format(username=msg["sender_full_name"],
message=msg["content"])
slack_text = SLACK_MESSAGE_TEMPLATE.format(
username=msg["sender_full_name"], message=msg["content"]
)
self.slack_webclient.chat_postMessage(
channel=self.channel,
text=slack_text,
)
return _zulip_to_slack
def run_slack_listener(self) -> None:
members = self.slack_webclient.users_list()['members']
# See also https://api.slack.com/changelog/2017-09-the-one-about-usernames
self.slack_id_to_name = {u["id"]: u["profile"].get("display_name", u["profile"]["real_name"]) for u in members}
self.slack_id_to_name = {
u["id"]: u["profile"].get("display_name", u["profile"]["real_name"]) for u in members
}
self.slack_name_to_id = {v: k for k, v in self.slack_id_to_name.items()}
@RTMClient.run_on(event='message')
@ -96,14 +103,13 @@ class SlackBridge:
self.replace_slack_id_with_name(msg)
content = ZULIP_MESSAGE_TEMPLATE.format(username=user, message=msg['text'])
msg_data = dict(
type="stream",
to=self.zulip_stream,
subject=self.zulip_subject,
content=content)
type="stream", to=self.zulip_stream, subject=self.zulip_subject, content=content
)
self.zulip_client.send_message(msg_data)
self.slack_client.start()
if __name__ == "__main__":
usage = """run-slack-bridge
@ -124,7 +130,9 @@ if __name__ == "__main__":
try:
sb = SlackBridge(config)
zp = threading.Thread(target=sb.zulip_client.call_on_each_message, args=(sb.zulip_to_slack(),))
zp = threading.Thread(
target=sb.zulip_client.call_on_each_message, args=(sb.zulip_to_slack(),)
)
sp = threading.Thread(target=sb.run_slack_listener, args=())
print("Starting message handler on Zulip client")
zp.start()

109
zulip/integrations/codebase/zulip_codebase_mirror
View file

@ -39,7 +39,8 @@ client = zulip.Client(
email=config.ZULIP_USER,
site=config.ZULIP_SITE,
api_key=config.ZULIP_API_KEY,
client="ZulipCodebase/" + VERSION)
client="ZulipCodebase/" + VERSION,
)
user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
# find some form of JSON loader/dumper, with a preference order for speed.
@ -52,13 +53,18 @@ while len(json_implementations):
except ImportError:
continue
def make_api_call(path: str) -> Optional[List[Dict[str, Any]]]:
response = requests.get("https://api3.codebasehq.com/%s" % (path,),
auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
params={'raw': 'True'},
headers = {"User-Agent": user_agent,
"Content-Type": "application/json",
"Accept": "application/json"})
response = requests.get(
"https://api3.codebasehq.com/%s" % (path,),
auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
params={'raw': 'True'},
headers={
"User-Agent": user_agent,
"Content-Type": "application/json",
"Accept": "application/json",
},
)
if response.status_code == 200:
return json.loads(response.text)
@ -69,12 +75,16 @@ def make_api_call(path: str) -> Optional[List[Dict[str, Any]]]:
logging.error("Bad authorization from Codebase. Please check your credentials")
sys.exit(-1)
else:
logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
logging.warn(
"Found non-success response status code: %s %s" % (response.status_code, response.text)
)
return None
def make_url(path: str) -> str:
return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
def handle_event(event: Dict[str, Any]) -> None:
event = event['event']
event_type = event['type']
@ -114,11 +124,17 @@ def handle_event(event: Dict[str, Any]) -> None:
else:
if new_ref:
branch = "new branch %s" % (branch,)
content = ("%s pushed %s commit(s) to %s in project %s:\n\n" %
(actor_name, num_commits, branch, project))
content = "%s pushed %s commit(s) to %s in project %s:\n\n" % (
actor_name,
num_commits,
branch,
project,
)
for commit in raw_props.get('commits'):
ref = commit.get('ref')
url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
url = make_url(
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref)
)
message = commit.get('message')
content += "* [%s](%s): %s\n" % (ref, url, message)
elif event_type == 'ticketing_ticket':
@ -133,8 +149,10 @@ def handle_event(event: Dict[str, Any]) -> None:
if assignee is None:
assignee = "no one"
subject = "#%s: %s" % (num, name)
content = ("""%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" %
(actor_name, num, url, priority, assignee, name))
content = (
"""%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s"""
% (actor_name, num, url, priority, assignee, name)
)
elif event_type == 'ticketing_note':
stream = config.ZULIP_TICKETS_STREAM_NAME
@ -148,11 +166,19 @@ def handle_event(event: Dict[str, Any]) -> None:
content = ""
if body is not None and len(body) > 0:
content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (
actor_name,
num,
url,
body,
)
if 'status_id' in changes:
status_change = changes.get('status_id')
content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
content += "Status changed from **%s** to **%s**\n\n" % (
status_change[0],
status_change[1],
)
elif event_type == 'ticketing_milestone':
stream = config.ZULIP_TICKETS_STREAM_NAME
@ -172,10 +198,17 @@ def handle_event(event: Dict[str, Any]) -> None:
if commit:
repo_link = raw_props.get('repository_permalink')
url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
url = make_url(
'projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit)
)
subject = "%s commented on %s" % (actor_name, commit)
content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (
actor_name,
commit,
url,
comment,
)
else:
# Otherwise, this is a Discussion item, and handle it
subj = raw_props.get("subject")
@ -199,17 +232,32 @@ def handle_event(event: Dict[str, Any]) -> None:
servers = raw_props.get('servers')
repo_link = raw_props.get('repository_permalink')
start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (
project_link, repo_link, start_ref, end_ref))
start_ref_url = make_url(
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref)
)
end_ref_url = make_url(
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref)
)
between_url = make_url(
"projects/%s/repositories/%s/compare/%s...%s"
% (project_link, repo_link, start_ref, end_ref)
)
subject = "Deployment to %s" % (environment,)
content = ("%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." %
(actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment))
content = "%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." % (
actor_name,
start_ref,
start_ref_url,
between_url,
end_ref,
end_ref_url,
environment,
)
if servers is not None:
content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
content += "\n\nServers deployed to: %s" % (
", ".join(["`%s`" % (server,) for server in servers])
)
elif event_type == 'named_tree':
# Docs say named_tree type used for new/deleting branches and tags,
@ -228,10 +276,9 @@ def handle_event(event: Dict[str, Any]) -> None:
if len(subject) > 60:
subject = subject[:57].rstrip() + '...'
res = client.send_message({"type": "stream",
"to": stream,
"subject": subject,
"content": content})
res = client.send_message(
{"type": "stream", "to": stream, "subject": subject, "content": content}
)
if res['result'] == 'success':
logging.info("Successfully sent Zulip with id: %s" % (res['id'],))
else:
@ -278,6 +325,7 @@ def run_mirror() -> None:
open(config.RESUME_FILE, 'w').write(since.strftime("%s"))
logging.info("Shutting down Codebase mirror")
# void function that checks the permissions of the files this script needs.
def check_permissions() -> None:
# check that the log file can be written
@ -291,9 +339,12 @@ def check_permissions() -> None:
try:
open(config.RESUME_FILE, "a+")
except OSError as e:
sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
sys.stderr.write(
"Could not open up the file %s for reading and writing" % (config.RESUME_FILE,)
)
sys.stderr.write(str(e))
if __name__ == "__main__":
assert isinstance(config.RESUME_FILE, str), "RESUME_FILE path not given; refusing to continue"
check_permissions()

20
zulip/integrations/git/post-receive
View file

@ -29,30 +29,33 @@ client = zulip.Client(
email=config.ZULIP_USER,
site=config.ZULIP_SITE,
api_key=config.ZULIP_API_KEY,
client="ZulipGit/" + VERSION)
client="ZulipGit/" + VERSION,
)
def git_repository_name() -> Text:
output = subprocess.check_output(["git", "rev-parse", "--is-bare-repository"])
if output.strip() == "true":
return os.path.basename(os.getcwd())[:-len(".git")]
return os.path.basename(os.getcwd())[: -len(".git")]
else:
return os.path.basename(os.path.dirname(os.getcwd()))
def git_commit_range(oldrev: str, newrev: str) -> str:
log_cmd = ["git", "log", "--reverse",
"--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
log_cmd = ["git", "log", "--reverse", "--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
commits = ''
for ln in subprocess.check_output(log_cmd, universal_newlines=True).splitlines():
author_email, commit_id, subject = ln.split(None, 2)
author_email, commit_id, subject = ln.split(None, 2)
if hasattr(config, "format_commit_message"):
commits += config.format_commit_message(author_email, subject, commit_id)
else:
commits += '!avatar(%s) %s\n' % (author_email, subject)
return commits
def send_bot_message(oldrev: str, newrev: str, refname: str) -> None:
repo_name = git_repository_name()
branch = refname.replace('refs/heads/', '')
repo_name = git_repository_name()
branch = refname.replace('refs/heads/', '')
destination = config.commit_notice_destination(repo_name, branch, newrev)
if destination is None:
# Don't forward the notice anywhere
@ -69,7 +72,7 @@ def send_bot_message(oldrev: str, newrev: str, refname: str) -> None:
added = ''
removed = ''
else:
added = git_commit_range(oldrev, newrev)
added = git_commit_range(oldrev, newrev)
removed = git_commit_range(newrev, oldrev)
if oldrev == '0000000000000000000000000000000000000000':
@ -95,6 +98,7 @@ def send_bot_message(oldrev: str, newrev: str, refname: str) -> None:
}
client.send_message(message_data)
for ln in sys.stdin:
oldrev, newrev, refname = ln.strip().split()
send_bot_message(oldrev, newrev, refname)

5
zulip/integrations/git/zulip_git_config.py
View file

@ -25,12 +25,12 @@ ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
# And similarly for branch "test-post-receive" (for use when testing).
def commit_notice_destination(repo: Text, branch: Text, commit: Text) -> Optional[Dict[Text, Text]]:
if branch in ["master", "test-post-receive"]:
return dict(stream = STREAM_NAME,
subject = "%s" % (branch,))
return dict(stream=STREAM_NAME, subject="%s" % (branch,))
# Return None for cases where you don't want a notice sent
return None
# Modify this function to change how commits are displayed; the most
# common customization is to include a link to the commit in your
# graphical repository viewer, e.g.
@ -39,6 +39,7 @@ def commit_notice_destination(repo: Text, branch: Text, commit: Text) -> Optiona
def format_commit_message(author: Text, subject: Text, commit_id: Text) -> Text:
return '!avatar(%s) %s\n' % (author, subject)
## If properly installed, the Zulip API should be in your import
## path, but if not, set a custom path below
ZULIP_API_PATH: Optional[str] = None

10
zulip/integrations/google/get-google-credentials
View file

@ -7,7 +7,10 @@ from oauth2client.file import Storage
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args() # type: Optional[argparse.Namespace]
flags = argparse.ArgumentParser(
parents=[tools.argparser]
).parse_args() # type: Optional[argparse.Namespace]
except ImportError:
flags = None
@ -22,6 +25,7 @@ CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Zulip Calendar Bot'
HOME_DIR = os.path.expanduser('~')
def get_credentials() -> client.Credentials:
"""Gets valid user credentials from storage.
@ -32,8 +36,7 @@ def get_credentials() -> client.Credentials:
Credentials, the obtained credential.
"""
credential_path = os.path.join(HOME_DIR,
'google-credentials.json')
credential_path = os.path.join(HOME_DIR, 'google-credentials.json')
store = Storage(credential_path)
credentials = store.get()
@ -49,4 +52,5 @@ def get_credentials() -> client.Credentials:
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
get_credentials()

64
zulip/integrations/google/google-calendar
View file

@ -38,7 +38,9 @@ sent = set() # type: Set[Tuple[int, datetime.datetime]]
sys.path.append(os.path.dirname(__file__))
parser = zulip.add_default_arguments(argparse.ArgumentParser(r"""
parser = zulip.add_default_arguments(
argparse.ArgumentParser(
r"""
google-calendar --calendar calendarID@example.calendar.google.com
@ -53,23 +55,29 @@ google-calendar --calendar calendarID@example.calendar.google.com
revealed to local users through the command line.
Depends on: google-api-python-client
"""))
"""
)
)
parser.add_argument('--interval',
dest='interval',
default=30,
type=int,
action='store',
help='Minutes before event for reminder [default: 30]',
metavar='MINUTES')
parser.add_argument(
'--interval',
dest='interval',
default=30,
type=int,
action='store',
help='Minutes before event for reminder [default: 30]',
metavar='MINUTES',
)
parser.add_argument('--calendar',
dest = 'calendarID',
default = 'primary',
type = str,
action = 'store',
help = 'Calendar ID for the calendar you want to receive reminders from.')
parser.add_argument(
'--calendar',
dest='calendarID',
default='primary',
type=str,
action='store',
help='Calendar ID for the calendar you want to receive reminders from.',
)
options = parser.parse_args()
@ -78,6 +86,7 @@ if not (options.zulip_email):
zulip_client = zulip.init_from_options(options)
def get_credentials() -> client.Credentials:
"""Gets valid user credentials from storage.
@ -89,8 +98,7 @@ def get_credentials() -> client.Credentials:
Credentials, the obtained credential.
"""
try:
credential_path = os.path.join(HOME_DIR,
'google-credentials.json')
credential_path = os.path.join(HOME_DIR, 'google-credentials.json')
store = Storage(credential_path)
credentials = store.get()
@ -110,8 +118,17 @@ def populate_events() -> Optional[None]:
service = discovery.build('calendar', 'v3', http=creds)
now = datetime.datetime.now(pytz.utc).isoformat()
feed = service.events().list(calendarId=options.calendarID, timeMin=now, maxResults=5,
singleEvents=True, orderBy='startTime').execute()
feed = (
service.events()
.list(
calendarId=options.calendarID,
timeMin=now,
maxResults=5,
singleEvents=True,
orderBy='startTime',
)
.execute()
)
events = []
for event in feed["items"]:
@ -172,14 +189,13 @@ def send_reminders() -> Optional[None]:
else:
message = 'Reminder:\n\n' + '\n'.join('* ' + m for m in messages)
zulip_client.send_message(dict(
type = 'private',
to = options.zulip_email,
sender = options.zulip_email,
content = message))
zulip_client.send_message(
dict(type='private', to=options.zulip_email, sender=options.zulip_email, content=message)
)
sent.update(keys)
# Loop forever
for i in itertools.count():
try:

35
zulip/integrations/hg/zulip_changegroup.py
View file

@ -15,7 +15,10 @@ import zulip
VERSION = "0.9"
def format_summary_line(web_url: str, user: str, base: int, tip: int, branch: str, node: Text) -> Text:
def format_summary_line(
web_url: str, user: str, base: int, tip: int, branch: str, node: Text
) -> Text:
"""
Format the first line of the message, which contains summary
information about the changeset and links to the changelog if a
@ -29,16 +32,18 @@ def format_summary_line(web_url: str, user: str, base: int, tip: int, branch: st
if web_url:
shortlog_base_url = web_url.rstrip("/") + "/shortlog/"
summary_url = "{shortlog}{tip}?revcount={revcount}".format(
shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount)
shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount
)
formatted_commit_count = "[{revcount} commit{s}]({url})".format(
revcount=revcount, s=plural, url=summary_url)
revcount=revcount, s=plural, url=summary_url
)
else:
formatted_commit_count = "{revcount} commit{s}".format(
revcount=revcount, s=plural)
formatted_commit_count = "{revcount} commit{s}".format(revcount=revcount, s=plural)
return "**{user}** pushed {commits} to **{branch}** (`{tip}:{node}`):\n\n".format(
user=user, commits=formatted_commit_count, branch=branch, tip=tip,
node=node[:12])
user=user, commits=formatted_commit_count, branch=branch, tip=tip, node=node[:12]
)
def format_commit_lines(web_url: str, repo: repo, base: int, tip: int) -> str:
"""
@ -56,8 +61,7 @@ def format_commit_lines(web_url: str, repo: repo, base: int, tip: int) -> str:
if web_url:
summary_url = rev_base_url + str(rev_ctx)
summary = "* [{summary}]({url})".format(
summary=one_liner, url=summary_url)
summary = "* [{summary}]({url})".format(summary=one_liner, url=summary_url)
else:
summary = "* {summary}".format(summary=one_liner)
@ -65,14 +69,17 @@ def format_commit_lines(web_url: str, repo: repo, base: int, tip: int) -> str:
return "\n".join(summary for summary in commit_summaries)
def send_zulip(email: str, api_key: str, site: str, stream: str, subject: str, content: Text) -> None:
def send_zulip(
email: str, api_key: str, site: str, stream: str, subject: str, content: Text
) -> None:
"""
Send a message to Zulip using the provided credentials, which should be for
a bot in most cases.
"""
client = zulip.Client(email=email, api_key=api_key,
site=site,
client="ZulipMercurial/" + VERSION)
client = zulip.Client(
email=email, api_key=api_key, site=site, client="ZulipMercurial/" + VERSION
)
message_data = {
"type": "stream",
@ -83,6 +90,7 @@ def send_zulip(email: str, api_key: str, site: str, stream: str, subject: str, c
client.send_message(message_data)
def get_config(ui: ui, item: str) -> str:
try:
# config returns configuration value.
@ -91,6 +99,7 @@ def get_config(ui: ui, item: str) -> str:
ui.warn("Zulip: Could not find required item {} in hg config.".format(item))
sys.exit(1)
def hook(ui: ui, repo: repo, **kwargs: Text) -> None:
"""
Invoked by configuring a [hook] entry in .hg/hgrc.

1
zulip/integrations/jabber/jabber_mirror.py
View file

@ -14,6 +14,7 @@ def die(signal: int, frame: FrameType) -> None:
"""We actually want to exit, so run os._exit (so as not to be caught and restarted)"""
os._exit(1)
signal.signal(signal.SIGINT, die)
args = [os.path.join(os.path.dirname(sys.argv[0]), "jabber_mirror_backend.py")]

166
zulip/integrations/jabber/jabber_mirror_backend.py
View file

@ -52,18 +52,22 @@ from zulip import Client
__version__ = "1.1"
def room_to_stream(room: str) -> str:
return room + "/xmpp"
def stream_to_room(stream: str) -> str:
return stream.lower().rpartition("/xmpp")[0]
def jid_to_zulip(jid: JID) -> str:
suffix = ''
if not jid.username.endswith("-bot"):
suffix = options.zulip_email_suffix
return "%s%s@%s" % (jid.username, suffix, options.zulip_domain)
def zulip_to_jid(email: str, jabber_domain: str) -> JID:
jid = JID(email, domain=jabber_domain)
if (
@ -74,6 +78,7 @@ def zulip_to_jid(email: str, jabber_domain: str) -> JID:
jid.username = jid.username.rpartition(options.zulip_email_suffix)[0]
return jid
class JabberToZulipBot(ClientXMPP):
def __init__(self, jid: JID, password: str, rooms: List[str]) -> None:
if jid.resource:
@ -153,10 +158,10 @@ class JabberToZulipBot(ClientXMPP):
recipient = jid_to_zulip(msg["to"])
zulip_message = dict(
sender = sender,
type = "private",
to = recipient,
content = msg["body"],
sender=sender,
type="private",
to=recipient,
content=msg["body"],
)
ret = self.zulipToJabber.client.send_message(zulip_message)
if ret.get("result") != "success":
@ -178,12 +183,12 @@ class JabberToZulipBot(ClientXMPP):
jid = self.nickname_to_jid(msg.get_mucroom(), sender_nick)
sender = jid_to_zulip(jid)
zulip_message = dict(
forged = "yes",
sender = sender,
type = "stream",
subject = subject,
to = stream,
content = msg["body"],
forged="yes",
sender=sender,
type="stream",
subject=subject,
to=stream,
content=msg["body"],
)
ret = self.zulipToJabber.client.send_message(zulip_message)
if ret.get("result") != "success":
@ -191,11 +196,12 @@ class JabberToZulipBot(ClientXMPP):
def nickname_to_jid(self, room: str, nick: str) -> JID:
jid = self.plugin['xep_0045'].getJidProperty(room, nick, "jid")
if (jid is None or jid == ''):
if jid is None or jid == '':
return JID(local=nick.replace(' ', ''), domain=self.boundjid.domain)
else:
return jid
class ZulipToJabberBot:
def __init__(self, zulip_client: Client) -> None:
self.client = zulip_client
@ -221,7 +227,7 @@ class ZulipToJabberBot:
self.process_subscription(event)
def stream_message(self, msg: Dict[str, str]) -> None:
assert(self.jabber is not None)
assert self.jabber is not None
stream = msg['display_recipient']
if not stream.endswith("/xmpp"):
return
@ -229,14 +235,13 @@ class ZulipToJabberBot:
room = stream_to_room(stream)
jabber_recipient = JID(local=room, domain=options.conference_domain)
outgoing = self.jabber.make_message(
mto = jabber_recipient,
mbody = msg['content'],
mtype = 'groupchat')
mto=jabber_recipient, mbody=msg['content'], mtype='groupchat'
)
outgoing['thread'] = '\u1FFFE'
outgoing.send()
def private_message(self, msg: Dict[str, Any]) -> None:
assert(self.jabber is not None)
assert self.jabber is not None
for recipient in msg['display_recipient']:
if recipient["email"] == self.client.email:
continue
@ -245,14 +250,13 @@ class ZulipToJabberBot:
recip_email = recipient['email']
jabber_recipient = zulip_to_jid(recip_email, self.jabber.boundjid.domain)
outgoing = self.jabber.make_message(
mto = jabber_recipient,
mbody = msg['content'],
mtype = 'chat')
mto=jabber_recipient, mbody=msg['content'], mtype='chat'
)
outgoing['thread'] = '\u1FFFE'
outgoing.send()
def process_subscription(self, event: Dict[str, Any]) -> None:
assert(self.jabber is not None)
assert self.jabber is not None
if event['op'] == 'add':
streams = [s['name'].lower() for s in event['subscriptions']]
streams = [s for s in streams if s.endswith("/xmpp")]
@ -264,6 +268,7 @@ class ZulipToJabberBot:
for stream in streams:
self.jabber.leave_muc(stream_to_room(stream))
def get_rooms(zulipToJabber: ZulipToJabberBot) -> List[str]:
def get_stream_infos(key: str, method: Callable[[], Dict[str, Any]]) -> Any:
ret = method()
@ -284,17 +289,21 @@ def get_rooms(zulipToJabber: ZulipToJabberBot) -> List[str]:
rooms.append(stream_to_room(stream))
return rooms
def config_error(msg: str) -> None:
sys.stderr.write("%s\n" % (msg,))
sys.exit(2)
if __name__ == '__main__':
parser = optparse.OptionParser(
epilog='''Most general and Jabber configuration options may also be specified in the
zulip configuration file under the jabber_mirror section (exceptions are noted
in their help sections). Keys have the same name as options with hyphens
replaced with underscores. Zulip configuration options go in the api section,
as normal.'''.replace("\n", " ")
as normal.'''.replace(
"\n", " "
)
)
parser.add_option(
'--mode',
@ -305,7 +314,10 @@ as normal.'''.replace("\n", " ")
all messages they send on Zulip to Jabber and all private Jabber messages to
Zulip. In "public" mode, the mirror uses the credentials for a dedicated mirror
user and mirrors messages sent to Jabber rooms to Zulip. Defaults to
"personal"'''.replace("\n", " "))
"personal"'''.replace(
"\n", " "
),
)
parser.add_option(
'--zulip-email-suffix',
default=None,
@ -315,13 +327,19 @@ from JIDs and nicks before sending requests to the Zulip server, and remove the
suffix before sending requests to the Jabber server. For example, specifying
"+foo" will cause messages that are sent to the "bar" room by nickname "qux" to
be mirrored to the "bar/xmpp" stream in Zulip by user "qux+foo@example.com". This
option does not affect login credentials.'''.replace("\n", " "))
parser.add_option('-d', '--debug',
help='set logging to DEBUG. Can not be set via config file.',
action='store_const',
dest='log_level',
const=logging.DEBUG,
default=logging.INFO)
option does not affect login credentials.'''.replace(
"\n", " "
),
)
parser.add_option(
'-d',
'--debug',
help='set logging to DEBUG. Can not be set via config file.',
action='store_const',
dest='log_level',
const=logging.DEBUG,
default=logging.INFO,
)
jabber_group = optparse.OptionGroup(parser, "Jabber configuration")
jabber_group.add_option(
@ -329,39 +347,42 @@ option does not affect login credentials.'''.replace("\n", " "))
default=None,
action='store',
help="Your Jabber JID. If a resource is specified, "
"it will be used as the nickname when joining MUCs. "
"Specifying the nickname is mostly useful if you want "
"to run the public mirror from a regular user instead of "
"from a dedicated account.")
jabber_group.add_option('--jabber-password',
default=None,
action='store',
help="Your Jabber password")
jabber_group.add_option('--conference-domain',
default=None,
action='store',
help="Your Jabber conference domain (E.g. conference.jabber.example.com). "
"If not specifed, \"conference.\" will be prepended to your JID's domain.")
jabber_group.add_option('--no-use-tls',
default=None,
action='store_true')
jabber_group.add_option('--jabber-server-address',
default=None,
action='store',
help="The hostname of your Jabber server. This is only needed if "
"your server is missing SRV records")
jabber_group.add_option('--jabber-server-port',
default='5222',
action='store',
help="The port of your Jabber server. This is only needed if "
"your server is missing SRV records")
"it will be used as the nickname when joining MUCs. "
"Specifying the nickname is mostly useful if you want "
"to run the public mirror from a regular user instead of "
"from a dedicated account.",
)
jabber_group.add_option(
'--jabber-password', default=None, action='store', help="Your Jabber password"
)
jabber_group.add_option(
'--conference-domain',
default=None,
action='store',
help="Your Jabber conference domain (E.g. conference.jabber.example.com). "
"If not specifed, \"conference.\" will be prepended to your JID's domain.",
)
jabber_group.add_option('--no-use-tls', default=None, action='store_true')
jabber_group.add_option(
'--jabber-server-address',
default=None,
action='store',
help="The hostname of your Jabber server. This is only needed if "
"your server is missing SRV records",
)
jabber_group.add_option(
'--jabber-server-port',
default='5222',
action='store',
help="The port of your Jabber server. This is only needed if "
"your server is missing SRV records",
)
parser.add_option_group(jabber_group)
parser.add_option_group(zulip.generate_option_group(parser, "zulip-"))
(options, args) = parser.parse_args()
logging.basicConfig(level=options.log_level,
format='%(levelname)-8s %(message)s')
logging.basicConfig(level=options.log_level, format='%(levelname)-8s %(message)s')
if options.zulip_config_file is None:
default_config_file = zulip.get_default_config_filename()
@ -378,12 +399,16 @@ option does not affect login credentials.'''.replace("\n", " "))
config.readfp(f, config_file)
except OSError:
pass
for option in ("jid", "jabber_password", "conference_domain", "mode", "zulip_email_suffix",
"jabber_server_address", "jabber_server_port"):
if (
getattr(options, option) is None
and config.has_option("jabber_mirror", option)
):
for option in (
"jid",
"jabber_password",
"conference_domain",
"mode",
"zulip_email_suffix",
"jabber_server_address",
"jabber_server_port",
):
if getattr(options, option) is None and config.has_option("jabber_mirror", option):
setattr(options, option, config.get("jabber_mirror", option))
for option in ("no_use_tls",):
@ -403,10 +428,14 @@ option does not affect login credentials.'''.replace("\n", " "))
config_error("Bad value for --mode: must be one of 'public' or 'personal'")
if None in (options.jid, options.jabber_password):
config_error("You must specify your Jabber JID and Jabber password either "
"in the Zulip configuration file or on the commandline")
config_error(
"You must specify your Jabber JID and Jabber password either "
"in the Zulip configuration file or on the commandline"
)
zulipToJabber = ZulipToJabberBot(zulip.init_from_options(options, "JabberMirror/" + __version__))
zulipToJabber = ZulipToJabberBot(
zulip.init_from_options(options, "JabberMirror/" + __version__)
)
# This won't work for open realms that don't have a consistent domain
options.zulip_domain = zulipToJabber.client.email.partition('@')[-1]
@ -438,8 +467,9 @@ option does not affect login credentials.'''.replace("\n", " "))
try:
logging.info("Connecting to Zulip.")
zulipToJabber.client.call_on_each_event(zulipToJabber.process_event,
event_types=event_types)
zulipToJabber.client.call_on_each_event(
zulipToJabber.process_event, event_types=event_types
)
except BaseException:
logging.exception("Exception in main loop")
xmpp.abort()

21
zulip/integrations/log2zulip/log2zulip
View file

@ -14,10 +14,12 @@ import traceback
sys.path.append("/home/zulip/deployments/current")
try:
from scripts.lib.setup_path import setup_path
setup_path()
except ImportError:
try:
import scripts.lib.setup_path_on_import
scripts.lib.setup_path_on_import # Suppress unused import warning
except ImportError:
pass
@ -31,6 +33,7 @@ import zulip
temp_dir = "/var/tmp/" if os.name == "posix" else tempfile.gettempdir()
def mkdir_p(path: str) -> None:
# Python doesn't have an analog to `mkdir -p` < Python 3.2.
try:
@ -41,14 +44,18 @@ def mkdir_p(path: str) -> None:
else:
raise
def send_log_zulip(file_name: str, count: int, lines: List[str], extra: str = "") -> None:
content = "%s new errors%s:\n```\n%s\n```" % (count, extra, "\n".join(lines))
zulip_client.send_message({
"type": "stream",
"to": "logs",
"subject": "%s on %s" % (file_name, platform.node()),
"content": content,
})
zulip_client.send_message(
{
"type": "stream",
"to": "logs",
"subject": "%s on %s" % (file_name, platform.node()),
"content": content,
}
)
def process_lines(raw_lines: List[str], file_name: str) -> None:
lines = []
@ -65,6 +72,7 @@ def process_lines(raw_lines: List[str], file_name: str) -> None:
else:
send_log_zulip(file_name, len(lines), lines)
def process_logs() -> None:
data_file_path = os.path.join(temp_dir, "log2zulip.state")
mkdir_p(os.path.dirname(data_file_path))
@ -95,6 +103,7 @@ def process_logs() -> None:
new_data[log_file] = file_data
open(data_file_path, "w").write(json.dumps(new_data))
if __name__ == "__main__":
parser = zulip.add_default_arguments(argparse.ArgumentParser()) # type: argparse.ArgumentParser
parser.add_argument("--control-path", default="/etc/log2zulip.conf")

7
zulip/integrations/nagios/nagios-notify-zulip
View file

@ -17,8 +17,9 @@ for opt in ('type', 'host', 'service', 'state'):
parser.add_argument('--' + opt)
opts = parser.parse_args()
client = zulip.Client(config_file=opts.config,
client="ZulipNagios/" + VERSION) # type: zulip.Client
client = zulip.Client(
config_file=opts.config, client="ZulipNagios/" + VERSION
) # type: zulip.Client
msg = dict(type='stream', to=opts.stream) # type: Dict[str, Any]
@ -47,6 +48,6 @@ msg['content'] = '**%s**: %s is %s' % (opts.type, thing, opts.state)
output = (opts.output + '\n' + opts.long_output.replace(r'\n', '\n')).strip() # type: Text
if output:
# Put any command output in a code block.
msg['content'] += ('\n\n~~~~\n' + output + "\n~~~~\n")
msg['content'] += '\n\n~~~~\n' + output + "\n~~~~\n"
client.send_message(msg)

30
zulip/integrations/openshift/post_deploy
View file

@ -21,7 +21,9 @@ client = zulip.Client(
email=config.ZULIP_USER,
site=config.ZULIP_SITE,
api_key=config.ZULIP_API_KEY,
client='ZulipOpenShift/' + VERSION)
client='ZulipOpenShift/' + VERSION,
)
def get_deployment_details() -> Dict[str, str]:
# "gear deployments" output example:
@ -30,10 +32,13 @@ def get_deployment_details() -> Dict[str, str]:
dep = subprocess.check_output(['gear', 'deployments'], universal_newlines=True).splitlines()[1]
splits = dep.split(' - ')
return dict(app_name=os.environ['OPENSHIFT_APP_NAME'],
url=os.environ['OPENSHIFT_APP_DNS'],
branch=splits[2],
commit_id=splits[3])
return dict(
app_name=os.environ['OPENSHIFT_APP_NAME'],
url=os.environ['OPENSHIFT_APP_DNS'],
branch=splits[2],
commit_id=splits[3],
)
def send_bot_message(deployment: Dict[str, str]) -> None:
destination = config.deployment_notice_destination(deployment['branch'])
@ -42,14 +47,17 @@ def send_bot_message(deployment: Dict[str, str]) -> None:
return
message = config.format_deployment_message(**deployment)
client.send_message({
'type': 'stream',
'to': destination['stream'],
'subject': destination['subject'],
'content': message,
})
client.send_message(
{
'type': 'stream',
'to': destination['stream'],
'subject': destination['subject'],
'content': message,
}
)
return
deployment = get_deployment_details()
send_bot_message(deployment)

16
zulip/integrations/openshift/zulip_openshift_config.py
View file

@ -21,12 +21,12 @@ ZULIP_API_KEY = '0123456789abcdef0123456789abcdef'
# And similarly for branch "test-post-receive" (for use when testing).
def deployment_notice_destination(branch: str) -> Optional[Dict[str, Text]]:
if branch in ['master', 'test-post-receive']:
return dict(stream = 'deployments',
subject = '%s' % (branch,))
return dict(stream='deployments', subject='%s' % (branch,))
# Return None for cases where you don't want a notice sent
return None
# Modify this function to change how deployments are displayed
#
# It takes the following arguments:
@ -39,9 +39,15 @@ def deployment_notice_destination(branch: str) -> Optional[Dict[str, Text]]:
# * dep_id = deployment id
# * dep_time = deployment timestamp
def format_deployment_message(
app_name: str = '', url: str = '', branch: str = '', commit_id: str = '', dep_id: str = '', dep_time: str = '') -> str:
return 'Deployed commit `%s` (%s) in [%s](%s)' % (
commit_id, branch, app_name, url)
app_name: str = '',
url: str = '',
branch: str = '',
commit_id: str = '',
dep_id: str = '',
dep_time: str = '',
) -> str:
return 'Deployed commit `%s` (%s) in [%s](%s)' % (commit_id, branch, app_name, url)
## If properly installed, the Zulip API should be in your import
## path, but if not, set a custom path below

13
zulip/integrations/perforce/zulip_change-commit.py
View file

@ -36,7 +36,8 @@ client = zulip.Client(
email=config.ZULIP_USER,
site=config.ZULIP_SITE,
api_key=config.ZULIP_API_KEY,
client="ZulipPerforce/" + __version__) # type: zulip.Client
client="ZulipPerforce/" + __version__,
) # type: zulip.Client
try:
changelist = int(sys.argv[1]) # type: int
@ -52,7 +53,9 @@ except ValueError:
metadata = git_p4.p4_describe(changelist) # type: Dict[str, str]
destination = config.commit_notice_destination(changeroot, changelist) # type: Optional[Dict[str, str]]
destination = config.commit_notice_destination(
changeroot, changelist
) # type: Optional[Dict[str, str]]
if destination is None:
# Don't forward the notice anywhere
@ -84,10 +87,8 @@ message = """**{user}** committed revision @{change} to `{path}`.
{desc}
```
""".format(
user=metadata["user"],
change=change,
path=changeroot,
desc=metadata["desc"]) # type: str
user=metadata["user"], change=change, path=changeroot, desc=metadata["desc"]
) # type: str
message_data = {
"type": "stream",

4
zulip/integrations/perforce/zulip_perforce_config.py
View file

@ -37,12 +37,12 @@ def commit_notice_destination(path: Text, changelist: int) -> Optional[Dict[Text
directory = dirs[2]
if directory not in ["evil-master-plan", "my-super-secret-repository"]:
return dict(stream = "%s-commits" % (directory,),
subject = path)
return dict(stream="%s-commits" % (directory,), subject=path)
# Return None for cases where you don't want a notice sent
return None
## If properly installed, the Zulip API should be in your import
## path, but if not, set a custom path below
ZULIP_API_PATH: Optional[str] = None

122
zulip/integrations/rss/rss-bot
View file

@ -48,35 +48,48 @@ stream every 5 minutes is:
*/5 * * * * /usr/local/share/zulip/integrations/rss/rss-bot"""
parser = zulip.add_default_arguments(argparse.ArgumentParser(usage)) # type: argparse.ArgumentParser
parser.add_argument('--stream',
dest='stream',
help='The stream to which to send RSS messages.',
default="rss",
action='store')
parser.add_argument('--data-dir',
dest='data_dir',
help='The directory where feed metadata is stored',
default=os.path.join(RSS_DATA_DIR),
action='store')
parser.add_argument('--feed-file',
dest='feed_file',
help='The file containing a list of RSS feed URLs to follow, one URL per line',
default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
action='store')
parser.add_argument('--unwrap',
dest='unwrap',
action='store_true',
help='Convert word-wrapped paragraphs into single lines',
default=False)
parser.add_argument('--math',
dest='math',
action='store_true',
help='Convert $ to $$ (for KaTeX processing)',
default=False)
parser = zulip.add_default_arguments(
argparse.ArgumentParser(usage)
) # type: argparse.ArgumentParser
parser.add_argument(
'--stream',
dest='stream',
help='The stream to which to send RSS messages.',
default="rss",
action='store',
)
parser.add_argument(
'--data-dir',
dest='data_dir',
help='The directory where feed metadata is stored',
default=os.path.join(RSS_DATA_DIR),
action='store',
)
parser.add_argument(
'--feed-file',
dest='feed_file',
help='The file containing a list of RSS feed URLs to follow, one URL per line',
default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
action='store',
)
parser.add_argument(
'--unwrap',
dest='unwrap',
action='store_true',
help='Convert word-wrapped paragraphs into single lines',
default=False,
)
parser.add_argument(
'--math',
dest='math',
action='store_true',
help='Convert $ to $$ (for KaTeX processing)',
default=False,
)
opts = parser.parse_args() # type: Any
def mkdir_p(path: str) -> None:
# Python doesn't have an analog to `mkdir -p` < Python 3.2.
try:
@ -87,6 +100,7 @@ def mkdir_p(path: str) -> None:
else:
raise
try:
mkdir_p(opts.data_dir)
except OSError:
@ -106,11 +120,13 @@ logger = logging.getLogger(__name__) # type: logging.Logger
logger.setLevel(logging.DEBUG)
logger.addHandler(file_handler)
def log_error_and_exit(error: str) -> None:
logger.error(error)
logger.error(usage)
exit(1)
class MLStripper(HTMLParser):
def __init__(self) -> None:
super().__init__()
@ -123,57 +139,70 @@ class MLStripper(HTMLParser):
def get_data(self) -> str:
return ''.join(self.fed)
def strip_tags(html: str) -> str:
stripper = MLStripper()
stripper.feed(html)
return stripper.get_data()
def compute_entry_hash(entry: Dict[str, Any]) -> str:
entry_time = entry.get("published", entry.get("updated"))
entry_id = entry.get("id", entry.get("link"))
return hashlib.md5((entry_id + str(entry_time)).encode()).hexdigest()
def unwrap_text(body: str) -> str:
# Replace \n by space if it is preceded and followed by a non-\n.
# Example: '\na\nb\nc\n\nd\n' -> '\na b c\n\nd\n'
return re.sub('(?<=[^\n])\n(?=[^\n])', ' ', body)
def elide_subject(subject: str) -> str:
MAX_TOPIC_LENGTH = 60
if len(subject) > MAX_TOPIC_LENGTH:
subject = subject[:MAX_TOPIC_LENGTH - 3].rstrip() + '...'
subject = subject[: MAX_TOPIC_LENGTH - 3].rstrip() + '...'
return subject
def send_zulip(entry: Any, feed_name: str) -> Dict[str, Any]:
body = entry.summary # type: str
if opts.unwrap:
body = unwrap_text(body)
content = "**[%s](%s)**\n%s\n%s" % (entry.title,
entry.link,
strip_tags(body),
entry.link) # type: str
content = "**[%s](%s)**\n%s\n%s" % (
entry.title,
entry.link,
strip_tags(body),
entry.link,
) # type: str
if opts.math:
content = content.replace('$', '$$')
message = {"type": "stream",
"sender": opts.zulip_email,
"to": opts.stream,
"subject": elide_subject(feed_name),
"content": content,
} # type: Dict[str, str]
message = {
"type": "stream",
"sender": opts.zulip_email,
"to": opts.stream,
"subject": elide_subject(feed_name),
"content": content,
} # type: Dict[str, str]
return client.send_message(message)
try:
with open(opts.feed_file) as f:
feed_urls = [feed.strip() for feed in f.readlines()] # type: List[str]
except OSError:
log_error_and_exit("Unable to read feed file at %s." % (opts.feed_file,))
client = zulip.Client(email=opts.zulip_email, api_key=opts.zulip_api_key,
config_file=opts.zulip_config_file,
site=opts.zulip_site, client="ZulipRSS/" + VERSION) # type: zulip.Client
client = zulip.Client(
email=opts.zulip_email,
api_key=opts.zulip_api_key,
config_file=opts.zulip_config_file,
site=opts.zulip_site,
client="ZulipRSS/" + VERSION,
) # type: zulip.Client
first_message = True # type: bool
@ -182,7 +211,9 @@ for feed_url in feed_urls:
try:
with open(feed_file) as f:
old_feed_hashes = {line.strip(): True for line in f.readlines()} # type: Dict[str, bool]
old_feed_hashes = {
line.strip(): True for line in f.readlines()
} # type: Dict[str, bool]
except OSError:
old_feed_hashes = {}
@ -192,8 +223,13 @@ for feed_url in feed_urls:
for entry in data.entries:
entry_hash = compute_entry_hash(entry) # type: str
# An entry has either been published or updated.
entry_time = entry.get("published_parsed", entry.get("updated_parsed")) # type: Tuple[int, int]
if entry_time is not None and (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24:
entry_time = entry.get(
"published_parsed", entry.get("updated_parsed")
) # type: Tuple[int, int]
if (
entry_time is not None
and (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24
):
# As a safeguard against misbehaving feeds, don't try to process
# entries older than some threshold.
continue

13
zulip/integrations/svn/post-commit
View file

@ -30,7 +30,8 @@ client = zulip.Client(
email=config.ZULIP_USER,
site=config.ZULIP_SITE,
api_key=config.ZULIP_API_KEY,
client="ZulipSVN/" + VERSION) # type: zulip.Client
client="ZulipSVN/" + VERSION,
) # type: zulip.Client
svn = pysvn.Client() # type: pysvn.Client
path, rev = sys.argv[1:] # type: Tuple[Text, Text]
@ -38,12 +39,12 @@ path, rev = sys.argv[1:] # type: Tuple[Text, Text]
# since its a local path, prepend "file://"
path = "file://" + path
entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[0] # type: Dict[Text, Any]
entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[
0
] # type: Dict[Text, Any]
message = "**{}** committed revision r{} to `{}`.\n\n> {}".format(
entry['author'],
rev,
path.split('/')[-1],
entry['revprops']['svn:log']) # type: Text
entry['author'], rev, path.split('/')[-1], entry['revprops']['svn:log']
) # type: Text
destination = config.commit_notice_destination(path, rev) # type: Optional[Dict[Text, Text]]

4
zulip/integrations/svn/zulip_svn_config.py
View file

@ -21,12 +21,12 @@ ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
def commit_notice_destination(path: Text, commit: Text) -> Optional[Dict[Text, Text]]:
repo = path.split('/')[-1]
if repo not in ["evil-master-plan", "my-super-secret-repository"]:
return dict(stream = "commits",
subject = "%s" % (repo,))
return dict(stream="commits", subject="%s" % (repo,))
# Return None for cases where you don't want a notice sent
return None
## If properly installed, the Zulip API should be in your import
## path, but if not, set a custom path below
ZULIP_API_PATH: Optional[str] = None

50
zulip/integrations/trac/zulip_trac.py
View file

@ -33,38 +33,50 @@ client = zulip.Client(
email=config.ZULIP_USER,
site=config.ZULIP_SITE,
api_key=config.ZULIP_API_KEY,
client="ZulipTrac/" + VERSION)
client="ZulipTrac/" + VERSION,
)
def markdown_ticket_url(ticket: Any, heading: str = "ticket") -> str:
return "[%s #%s](%s/%s)" % (heading, ticket.id, config.TRAC_BASE_TICKET_URL, ticket.id)
def markdown_block(desc: str) -> str:
return "\n\n>" + "\n> ".join(desc.split("\n")) + "\n"
def truncate(string: str, length: int) -> str:
if len(string) <= length:
return string
return string[:length - 3] + "..."
return string[: length - 3] + "..."
def trac_subject(ticket: Any) -> str:
return truncate("#%s: %s" % (ticket.id, ticket.values.get("summary")), 60)
def send_update(ticket: Any, content: str) -> None:
client.send_message({
"type": "stream",
"to": config.STREAM_FOR_NOTIFICATIONS,
"content": content,
"subject": trac_subject(ticket)
})
client.send_message(
{
"type": "stream",
"to": config.STREAM_FOR_NOTIFICATIONS,
"content": content,
"subject": trac_subject(ticket),
}
)
class ZulipPlugin(Component):
implements(ITicketChangeListener)
def ticket_created(self, ticket: Any) -> None:
"""Called when a ticket is created."""
content = "%s created %s in component **%s**, priority **%s**:\n" % \
(ticket.values.get("reporter"), markdown_ticket_url(ticket),
ticket.values.get("component"), ticket.values.get("priority"))
content = "%s created %s in component **%s**, priority **%s**:\n" % (
ticket.values.get("reporter"),
markdown_ticket_url(ticket),
ticket.values.get("component"),
ticket.values.get("priority"),
)
# Include the full subject if it will be truncated
if len(ticket.values.get("summary")) > 60:
content += "**%s**\n" % (ticket.values.get("summary"),)
@ -72,7 +84,9 @@ class ZulipPlugin(Component):
content += "%s" % (markdown_block(ticket.values.get("description")),)
send_update(ticket, content)
def ticket_changed(self, ticket: Any, comment: str, author: str, old_values: Dict[str, Any]) -> None:
def ticket_changed(
self, ticket: Any, comment: str, author: str, old_values: Dict[str, Any]
) -> None:
"""Called when a ticket is modified.
`old_values` is a dictionary containing the previous values of the
@ -92,15 +106,19 @@ class ZulipPlugin(Component):
field_changes = []
for key, value in old_values.items():
if key == "description":
content += '- Changed %s from %s\n\nto %s' % (key, markdown_block(value),
markdown_block(ticket.values.get(key)))
content += '- Changed %s from %s\n\nto %s' % (
key,
markdown_block(value),
markdown_block(ticket.values.get(key)),
)
elif old_values.get(key) == "":
field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
elif ticket.values.get(key) == "":
field_changes.append('%s: **%s** => ""' % (key, old_values.get(key)))
else:
field_changes.append('%s: **%s** => **%s**' % (key, old_values.get(key),
ticket.values.get(key)))
field_changes.append(
'%s: **%s** => **%s**' % (key, old_values.get(key), ticket.values.get(key))
)
content += ", ".join(field_changes)
send_update(ticket, content)

72
zulip/integrations/trello/zulip_trello.py
View file

@ -13,6 +13,7 @@ except ImportError:
print("http://docs.python-requests.org/en/master/user/install/")
sys.exit(1)
def get_model_id(options):
"""get_model_id
@ -24,19 +25,14 @@ def get_model_id(options):
"""
trello_api_url = 'https://api.trello.com/1/board/{}'.format(
options.trello_board_id
)
trello_api_url = 'https://api.trello.com/1/board/{}'.format(options.trello_board_id)
params = {
'key': options.trello_api_key,
'token': options.trello_token,
}
trello_response = requests.get(
trello_api_url,
params=params
)
trello_response = requests.get(trello_api_url, params=params)
if trello_response.status_code != 200:
print('Error: Can\'t get the idModel. Please check the configuration')
@ -68,13 +64,10 @@ def get_webhook_id(options, id_model):
options.trello_board_name,
),
'callbackURL': options.zulip_webhook_url,
'idModel': id_model
'idModel': id_model,
}
trello_response = requests.post(
trello_api_url,
data=data
)
trello_response = requests.post(trello_api_url, data=data)
if trello_response.status_code != 200:
print('Error: Can\'t create the Webhook:', trello_response.text)
@ -84,6 +77,7 @@ def get_webhook_id(options, id_model):
return webhook_info_json['id']
def create_webhook(options):
"""create_webhook
@ -106,8 +100,12 @@ def create_webhook(options):
if id_webhook:
print('Success! The webhook ID is', id_webhook)
print('Success! The webhook for the {} Trello board was successfully created.'.format(
options.trello_board_name))
print(
'Success! The webhook for the {} Trello board was successfully created.'.format(
options.trello_board_name
)
)
def main():
description = """
@ -120,28 +118,36 @@ at <https://zulip.com/integrations/doc/trello>.
"""
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--trello-board-name',
required=True,
help='The Trello board name.')
parser.add_argument('--trello-board-id',
required=True,
help=('The Trello board short ID. Can usually be found '
'in the URL of the Trello board.'))
parser.add_argument('--trello-api-key',
required=True,
help=('Visit https://trello.com/1/appkey/generate to generate '
'an APPLICATION_KEY (need to be logged into Trello).'))
parser.add_argument('--trello-token',
required=True,
help=('Visit https://trello.com/1/appkey/generate and under '
'`Developer API Keys`, click on `Token` and generate '
'a Trello access token.'))
parser.add_argument('--zulip-webhook-url',
required=True,
help='The webhook URL that Trello will query.')
parser.add_argument('--trello-board-name', required=True, help='The Trello board name.')
parser.add_argument(
'--trello-board-id',
required=True,
help=('The Trello board short ID. Can usually be found ' 'in the URL of the Trello board.'),
)
parser.add_argument(
'--trello-api-key',
required=True,
help=(
'Visit https://trello.com/1/appkey/generate to generate '
'an APPLICATION_KEY (need to be logged into Trello).'
),
)
parser.add_argument(
'--trello-token',
required=True,
help=(
'Visit https://trello.com/1/appkey/generate and under '
'`Developer API Keys`, click on `Token` and generate '
'a Trello access token.'
),
)
parser.add_argument(
'--zulip-webhook-url', required=True, help='The webhook URL that Trello will query.'
)
options = parser.parse_args()
create_webhook(options)
if __name__ == '__main__':
main()

74
zulip/integrations/twitter/twitter-bot
View file

@ -67,37 +67,34 @@ Make sure to go the application you created and click "create my
access token" as well. Fill in the values displayed.
"""
def write_config(config: ConfigParser, configfile_path: str) -> None:
with open(configfile_path, 'w') as configfile:
config.write(configfile)
parser = zulip.add_default_arguments(argparse.ArgumentParser("Fetch tweets from Twitter."))
parser.add_argument('--instructions',
action='store_true',
help='Show instructions for the twitter bot setup and exit'
)
parser.add_argument('--limit-tweets',
default=15,
type=int,
help='Maximum number of tweets to send at once')
parser.add_argument('--search',
dest='search_terms',
help='Terms to search on',
action='store')
parser.add_argument('--stream',
dest='stream',
help='The stream to which to send tweets',
default="twitter",
action='store')
parser.add_argument('--twitter-name',
dest='twitter_name',
help='Twitter username to poll new tweets from"')
parser.add_argument('--excluded-terms',
dest='excluded_terms',
help='Terms to exclude tweets on')
parser.add_argument('--excluded-users',
dest='excluded_users',
help='Users to exclude tweets on')
parser.add_argument(
'--instructions',
action='store_true',
help='Show instructions for the twitter bot setup and exit',
)
parser.add_argument(
'--limit-tweets', default=15, type=int, help='Maximum number of tweets to send at once'
)
parser.add_argument('--search', dest='search_terms', help='Terms to search on', action='store')
parser.add_argument(
'--stream',
dest='stream',
help='The stream to which to send tweets',
default="twitter",
action='store',
)
parser.add_argument(
'--twitter-name', dest='twitter_name', help='Twitter username to poll new tweets from"'
)
parser.add_argument('--excluded-terms', dest='excluded_terms', help='Terms to exclude tweets on')
parser.add_argument('--excluded-users', dest='excluded_users', help='Users to exclude tweets on')
opts = parser.parse_args()
@ -150,18 +147,22 @@ try:
except ImportError:
parser.error("Please install python-twitter")
api = twitter.Api(consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_token_key,
access_token_secret=access_token_secret)
api = twitter.Api(
consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_token_key,
access_token_secret=access_token_secret,
)
user = api.VerifyCredentials()
if not user.id:
print("Unable to log in to twitter with supplied credentials. Please double-check and try again")
print(
"Unable to log in to twitter with supplied credentials. Please double-check and try again"
)
sys.exit(1)
client = zulip.init_from_options(opts, client=client_type+VERSION)
client = zulip.init_from_options(opts, client=client_type + VERSION)
if opts.search_terms:
search_query = " OR ".join(opts.search_terms.split(","))
@ -190,7 +191,7 @@ if opts.excluded_users:
else:
excluded_users = []
for status in statuses[::-1][:opts.limit_tweets]:
for status in statuses[::-1][: opts.limit_tweets]:
# Check if the tweet is from an excluded user
exclude = False
for user in excluded_users:
@ -237,12 +238,7 @@ for status in statuses[::-1][:opts.limit_tweets]:
elif opts.twitter_name:
subject = composed
message = {
"type": "stream",
"to": [opts.stream],
"subject": subject,
"content": url
}
message = {"type": "stream", "to": [opts.stream], "subject": subject, "content": url}
ret = client.send_message(message)

108
zulip/integrations/zephyr/check-mirroring
View file

@ -13,36 +13,25 @@ import zephyr
import zulip
parser = optparse.OptionParser()
parser.add_option('--verbose',
dest='verbose',
default=False,
action='store_true')
parser.add_option('--site',
dest='site',
default=None,
action='store')
parser.add_option('--sharded',
default=False,
action='store_true')
parser.add_option('--verbose', dest='verbose', default=False, action='store_true')
parser.add_option('--site', dest='site', default=None, action='store')
parser.add_option('--sharded', default=False, action='store_true')
(options, args) = parser.parse_args()
mit_user = 'tabbott/extra@ATHENA.MIT.EDU'
zulip_client = zulip.Client(
verbose=True,
client="ZulipMonitoring/0.1",
site=options.site)
zulip_client = zulip.Client(verbose=True, client="ZulipMonitoring/0.1", site=options.site)
# Configure logging
log_file = "/var/log/zulip/check-mirroring-log"
log_format = "%(asctime)s: %(message)s"
log_file = "/var/log/zulip/check-mirroring-log"
log_format = "%(asctime)s: %(message)s"
logging.basicConfig(format=log_format)
formatter = logging.Formatter(log_format)
formatter = logging.Formatter(log_format)
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
logger = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(file_handler)
@ -75,13 +64,14 @@ if options.sharded:
for (stream, test) in test_streams:
if stream == "message":
continue
assert(hashlib.sha1(stream.encode("utf-8")).hexdigest().startswith(test))
assert hashlib.sha1(stream.encode("utf-8")).hexdigest().startswith(test)
else:
test_streams = [
("message", "p"),
("tabbott-nagios-test", "a"),
]
def print_status_and_exit(status: int) -> None:
# The output of this script is used by Nagios. Various outputs,
@ -91,6 +81,7 @@ def print_status_and_exit(status: int) -> None:
print(status)
sys.exit(status)
def send_zulip(message: Dict[str, str]) -> None:
result = zulip_client.send_message(message)
if result["result"] != "success":
@ -99,11 +90,16 @@ def send_zulip(message: Dict[str, str]) -> None:
logger.error(str(result))
print_status_and_exit(1)
# Returns True if and only if we "Detected server failure" sending the zephyr.
def send_zephyr(zwrite_args: List[str], content: str) -> bool:
p = subprocess.Popen(zwrite_args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
p = subprocess.Popen(
zwrite_args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
stdout, stderr = p.communicate(input=content)
if p.returncode != 0:
if "Detected server failure while receiving acknowledgement for" in stdout:
@ -116,6 +112,7 @@ def send_zephyr(zwrite_args: List[str], content: str) -> bool:
print_status_and_exit(1)
return False
# Subscribe to Zulip
try:
res = zulip_client.register(event_types=["message"])
@ -164,6 +161,8 @@ if not actually_subscribed:
# Prepare keys
zhkeys = {} # type: Dict[str, Tuple[str, str]]
hzkeys = {} # type: Dict[str, Tuple[str, str]]
def gen_key(key_dict: Dict[str, Tuple[str, str]]) -> str:
bits = str(random.getrandbits(32))
while bits in key_dict:
@ -171,10 +170,12 @@ def gen_key(key_dict: Dict[str, Tuple[str, str]]) -> str:
bits = str(random.getrandbits(32))
return bits
def gen_keys(key_dict: Dict[str, Tuple[str, str]]) -> None:
for (stream, test) in test_streams:
key_dict[gen_key(key_dict)] = (stream, test)
gen_keys(zhkeys)
gen_keys(hzkeys)
@ -196,6 +197,7 @@ def receive_zephyrs() -> None:
continue
notices.append(notice)
logger.info("Starting sending messages!")
# Send zephyrs
zsig = "Timothy Good Abbott"
@ -212,12 +214,15 @@ for key, (stream, test) in zhkeys.items():
zhkeys[new_key] = value
server_failure_again = send_zephyr(zwrite_args, str(new_key))
if server_failure_again:
logging.error("Zephyr server failure twice in a row on keys %s and %s! Aborting." %
(key, new_key))
logging.error(
"Zephyr server failure twice in a row on keys %s and %s! Aborting."
% (key, new_key)
)
print_status_and_exit(1)
else:
logging.warning("Replaced key %s with %s due to Zephyr server failure." %
(key, new_key))
logging.warning(
"Replaced key %s with %s due to Zephyr server failure." % (key, new_key)
)
receive_zephyrs()
receive_zephyrs()
@ -226,18 +231,22 @@ logger.info("Sent Zephyr messages!")
# Send Zulips
for key, (stream, test) in hzkeys.items():
if stream == "message":
send_zulip({
"type": "private",
"content": str(key),
"to": zulip_client.email,
})
send_zulip(
{
"type": "private",
"content": str(key),
"to": zulip_client.email,
}
)
else:
send_zulip({
"type": "stream",
"subject": "test",
"content": str(key),
"to": stream,
})
send_zulip(
{
"type": "stream",
"subject": "test",
"content": str(key),
"to": stream,
}
)
receive_zephyrs()
logger.info("Sent Zulip messages!")
@ -265,6 +274,8 @@ receive_zephyrs()
logger.info("Finished receiving Zephyr messages!")
all_keys = set(list(zhkeys.keys()) + list(hzkeys.keys()))
def process_keys(content_list: List[str]) -> Tuple[Dict[str, int], Set[str], Set[str], bool, bool]:
# Start by filtering out any keys that might have come from
@ -281,6 +292,7 @@ def process_keys(content_list: List[str]) -> Tuple[Dict[str, int], Set[str], Set
success = all(val == 1 for val in key_counts.values())
return key_counts, z_missing, h_missing, duplicates, success
# The h_foo variables are about the messages we _received_ in Zulip
# The z_foo variables are about the messages we _received_ in Zephyr
h_contents = [message["content"] for message in messages]
@ -302,12 +314,16 @@ for key in all_keys:
continue
if key in zhkeys:
(stream, test) = zhkeys[key]
logger.warning("%10s: z got %s, h got %s. Sent via Zephyr(%s): class %s" %
(key, z_key_counts[key], h_key_counts[key], test, stream))
logger.warning(
"%10s: z got %s, h got %s. Sent via Zephyr(%s): class %s"
% (key, z_key_counts[key], h_key_counts[key], test, stream)
)
if key in hzkeys:
(stream, test) = hzkeys[key]
logger.warning("%10s: z got %s. h got %s. Sent via Zulip(%s): class %s" %
(key, z_key_counts[key], h_key_counts[key], test, stream))
logger.warning(
"%10s: z got %s. h got %s. Sent via Zulip(%s): class %s"
% (key, z_key_counts[key], h_key_counts[key], test, stream)
)
logger.error("")
logger.error("Summary of specific problems:")
@ -322,10 +338,14 @@ if z_duplicates:
if z_missing_z:
logger.error("zephyr: Didn't receive all the Zephyrs we sent on the Zephyr end!")
logger.error("zephyr: This is probably an issue with check-mirroring sending or receiving Zephyrs.")
logger.error(
"zephyr: This is probably an issue with check-mirroring sending or receiving Zephyrs."
)
if h_missing_h:
logger.error("zulip: Didn't receive all the Zulips we sent on the Zulip end!")
logger.error("zulip: This is probably an issue with check-mirroring sending or receiving Zulips.")
logger.error(
"zulip: This is probably an issue with check-mirroring sending or receiving Zulips."
)
if z_missing_h:
logger.error("zephyr: Didn't receive all the Zulips we sent on the Zephyr end!")
if z_missing_h == h_missing_h:

4
zulip/integrations/zephyr/process_ccache
View file

@ -27,7 +27,9 @@ session_path = "/home/zulip/zephyr_sessions/%s" % (program_name,)
try:
if "--forward-mail-zephyrs" in open(supervisor_path).read():
template_data = template_data.replace("--use-sessions", "--use-sessions --forward-mail-zephyrs")
template_data = template_data.replace(
"--use-sessions", "--use-sessions --forward-mail-zephyrs"
)
except Exception:
pass
open(supervisor_path, "w").write(template_data.replace("USERNAME", short_user))

24
zulip/integrations/zephyr/sync-public-streams
View file

@ -17,9 +17,22 @@ def write_public_streams() -> None:
# Zephyr class names are canonicalized by first applying NFKC
# normalization and then lower-casing server-side
canonical_cls = unicodedata.normalize("NFKC", stream_name).lower()
if canonical_cls in ['security', 'login', 'network', 'ops', 'user_locate',
'mit', 'moof', 'wsmonitor', 'wg_ctl', 'winlogger',
'hm_ctl', 'hm_stat', 'zephyr_admin', 'zephyr_ctl']:
if canonical_cls in [
'security',
'login',
'network',
'ops',
'user_locate',
'mit',
'moof',
'wsmonitor',
'wg_ctl',
'winlogger',
'hm_ctl',
'hm_stat',
'zephyr_admin',
'zephyr_ctl',
]:
# These zephyr classes cannot be subscribed to by us, due
# to MIT's Zephyr access control settings
continue
@ -30,6 +43,7 @@ def write_public_streams() -> None:
f.write(json.dumps(list(public_streams)) + "\n")
os.rename("/home/zulip/public_streams.tmp", "/home/zulip/public_streams")
if __name__ == "__main__":
log_file = "/home/zulip/sync_public_streams.log"
logger = logging.getLogger(__name__)
@ -83,9 +97,7 @@ if __name__ == "__main__":
last_event_id = max(last_event_id, event["id"])
if event["type"] == "stream":
if event["op"] == "create":
stream_names.update(
stream["name"] for stream in event["streams"]
)
stream_names.update(stream["name"] for stream in event["streams"])
write_public_streams()
elif event["op"] == "delete":
stream_names.difference_update(

3
zulip/integrations/zephyr/zephyr_mirror.py
View file

@ -19,6 +19,7 @@ def die(signal: int, frame: FrameType) -> None:
# We actually want to exit, so run os._exit (so as not to be caught and restarted)
os._exit(1)
signal.signal(signal.SIGINT, die)
from zulip import RandomExponentialBackoff
@ -36,12 +37,14 @@ if options.forward_class_messages and not options.noshard:
if options.on_startup_command is not None:
subprocess.call([options.on_startup_command])
from zerver.lib.parallel import run_parallel
print("Starting parallel zephyr class mirroring bot")
jobs = list("0123456789abcdef")
def run_job(shard: str) -> int:
subprocess.call(args + ["--shard=%s" % (shard,)])
return 0
for (status, job) in run_parallel(run_job, jobs, threads=16):
print("A mirroring shard died!")
sys.exit(0)

517
zulip/integrations/zephyr/zephyr_mirror_backend.py
View file

@ -22,12 +22,16 @@ from zulip import RandomExponentialBackoff
DEFAULT_SITE = "https://api.zulip.com"
class States:
Startup, ZulipToZephyr, ZephyrToZulip, ChildSending = list(range(4))
CURRENT_STATE = States.Startup
logger: logging.Logger
def to_zulip_username(zephyr_username: str) -> str:
if "@" in zephyr_username:
(user, realm) = zephyr_username.split("@")
@ -40,6 +44,7 @@ def to_zulip_username(zephyr_username: str) -> str:
return user.lower() + "@mit.edu"
return user.lower() + "|" + realm.upper() + "@mit.edu"
def to_zephyr_username(zulip_username: str) -> str:
(user, realm) = zulip_username.split("@")
if "|" not in user:
@ -52,6 +57,7 @@ def to_zephyr_username(zulip_username: str) -> str:
raise Exception("Could not parse Zephyr realm for cross-realm user %s" % (zulip_username,))
return match_user.group(1).lower() + "@" + match_user.group(2).upper()
# Checks whether the pair of adjacent lines would have been
# linewrapped together, had they been intended to be parts of the same
# paragraph. Our check is whether if you move the first word on the
@ -70,6 +76,7 @@ def different_paragraph(line: str, next_line: str) -> bool:
or len(line) < len(words[0])
)
# Linewrapping algorithm based on:
# http://gcbenison.wordpress.com/2011/07/03/a-program-to-intelligently-remove-carriage-returns-so-you-can-paste-text-without-having-it-look-awful/ #ignorelongline
def unwrap_lines(body: str) -> str:
@ -78,9 +85,8 @@ def unwrap_lines(body: str) -> str:
previous_line = lines[0]
for line in lines[1:]:
line = line.rstrip()
if (
re.match(r'^\W', line, flags=re.UNICODE)
and re.match(r'^\W', previous_line, flags=re.UNICODE)
if re.match(r'^\W', line, flags=re.UNICODE) and re.match(
r'^\W', previous_line, flags=re.UNICODE
):
result += previous_line + "\n"
elif (
@ -99,6 +105,7 @@ def unwrap_lines(body: str) -> str:
result += previous_line
return result
class ZephyrDict(TypedDict, total=False):
type: Literal["private", "stream"]
time: str
@ -109,6 +116,7 @@ class ZephyrDict(TypedDict, total=False):
content: str
zsig: str
def send_zulip(zeph: ZephyrDict) -> Dict[str, Any]:
message: Dict[str, Any]
message = {}
@ -142,15 +150,20 @@ def send_zulip(zeph: ZephyrDict) -> Dict[str, Any]:
return zulip_client.send_message(message)
def send_error_zulip(error_msg: str) -> None:
message = {"type": "private",
"sender": zulip_account_email,
"to": zulip_account_email,
"content": error_msg,
}
message = {
"type": "private",
"sender": zulip_account_email,
"to": zulip_account_email,
"content": error_msg,
}
zulip_client.send_message(message)
current_zephyr_subs = set()
def zephyr_bulk_subscribe(subs: List[Tuple[str, str, str]]) -> None:
try:
zephyr._z.subAll(subs)
@ -186,6 +199,7 @@ def zephyr_bulk_subscribe(subs: List[Tuple[str, str, str]]) -> None:
else:
current_zephyr_subs.add(cls)
def update_subscriptions() -> None:
try:
f = open(options.stream_file_path)
@ -198,10 +212,9 @@ def update_subscriptions() -> None:
classes_to_subscribe = set()
for stream in public_streams:
zephyr_class = stream
if (
options.shard is not None
and not hashlib.sha1(zephyr_class.encode("utf-8")).hexdigest().startswith(options.shard)
):
if options.shard is not None and not hashlib.sha1(
zephyr_class.encode("utf-8")
).hexdigest().startswith(options.shard):
# This stream is being handled by a different zephyr_mirror job.
continue
if zephyr_class in current_zephyr_subs:
@ -211,6 +224,7 @@ def update_subscriptions() -> None:
if len(classes_to_subscribe) > 0:
zephyr_bulk_subscribe(list(classes_to_subscribe))
def maybe_kill_child() -> None:
try:
if child_pid is not None:
@ -219,10 +233,14 @@ def maybe_kill_child() -> None:
# We don't care if the child process no longer exists, so just log the error
logger.exception("")
def maybe_restart_mirroring_script() -> None:
if os.stat(os.path.join(options.stamp_path, "stamps", "restart_stamp")).st_mtime > start_time or (
if os.stat(
os.path.join(options.stamp_path, "stamps", "restart_stamp")
).st_mtime > start_time or (
(options.user == "tabbott" or options.user == "tabbott/extra")
and os.stat(os.path.join(options.stamp_path, "stamps", "tabbott_stamp")).st_mtime > start_time
and os.stat(os.path.join(options.stamp_path, "stamps", "tabbott_stamp")).st_mtime
> start_time
):
logger.warning("")
logger.warning("zephyr mirroring script has been updated; restarting...")
@ -244,6 +262,7 @@ def maybe_restart_mirroring_script() -> None:
backoff.fail()
raise Exception("Failed to reload too many times, aborting!")
def process_loop(log: Optional[IO[str]]) -> NoReturn:
restart_check_count = 0
last_check_time = time.time()
@ -287,6 +306,7 @@ def process_loop(log: Optional[IO[str]]) -> NoReturn:
except Exception:
logger.exception("Error updating subscriptions from Zulip:")
def parse_zephyr_body(zephyr_data: str, notice_format: str) -> Tuple[str, str]:
try:
(zsig, body) = zephyr_data.split("\x00", 1)
@ -298,13 +318,19 @@ def parse_zephyr_body(zephyr_data: str, notice_format: str) -> Tuple[str, str]:
fields = body.split('\x00')
if len(fields) == 5:
body = 'New transaction [%s] entered in %s\nFrom: %s (%s)\nSubject: %s' % (
fields[0], fields[1], fields[2], fields[4], fields[3])
fields[0],
fields[1],
fields[2],
fields[4],
fields[3],
)
except ValueError:
(zsig, body) = ("", zephyr_data)
# Clean body of any null characters, since they're invalid in our protocol.
body = body.replace('\x00', '')
return (zsig, body)
def parse_crypt_table(zephyr_class: str, instance: str) -> Optional[str]:
try:
crypt_table = open(os.path.join(os.environ["HOME"], ".crypt-table"))
@ -315,17 +341,23 @@ def parse_crypt_table(zephyr_class: str, instance: str) -> Optional[str]:
if line.strip() == "":
# Ignore blank lines
continue
match = re.match(r"^crypt-(?P<class>\S+):\s+((?P<algorithm>(AES|DES)):\s+)?(?P<keypath>\S+)$", line)
match = re.match(
r"^crypt-(?P<class>\S+):\s+((?P<algorithm>(AES|DES)):\s+)?(?P<keypath>\S+)$", line
)
if match is None:
# Malformed crypt_table line
logger.debug("Invalid crypt_table line!")
continue
groups = match.groupdict()
if groups['class'].lower() == zephyr_class and 'keypath' in groups and \
groups.get("algorithm") == "AES":
if (
groups['class'].lower() == zephyr_class
and 'keypath' in groups
and groups.get("algorithm") == "AES"
):
return groups["keypath"]
return None
def decrypt_zephyr(zephyr_class: str, instance: str, body: str) -> str:
keypath = parse_crypt_table(zephyr_class, instance)
if keypath is None:
@ -337,27 +369,32 @@ def decrypt_zephyr(zephyr_class: str, instance: str, body: str) -> str:
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
# decrypt the message!
p = subprocess.Popen(["gpg",
"--decrypt",
"--no-options",
"--no-default-keyring",
"--keyring=/dev/null",
"--secret-keyring=/dev/null",
"--batch",
"--quiet",
"--no-use-agent",
"--passphrase-file",
keypath],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
errors="replace")
p = subprocess.Popen(
[
"gpg",
"--decrypt",
"--no-options",
"--no-default-keyring",
"--keyring=/dev/null",
"--secret-keyring=/dev/null",
"--batch",
"--quiet",
"--no-use-agent",
"--passphrase-file",
keypath,
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
errors="replace",
)
decrypted, _ = p.communicate(input=body)
# Restore our ignoring signals
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
return decrypted
def process_notice(notice: "zephyr.ZNotice", log: Optional[IO[str]]) -> None:
assert notice.sender is not None
(zsig, body) = parse_zephyr_body(notice.message, notice.format)
@ -382,8 +419,7 @@ def process_notice(notice: "zephyr.ZNotice", log: Optional[IO[str]]) -> None:
if is_personal and not options.forward_personals:
return
if (zephyr_class not in current_zephyr_subs) and not is_personal:
logger.debug("Skipping ... %s/%s/%s" %
(zephyr_class, notice.instance, is_personal))
logger.debug("Skipping ... %s/%s/%s" % (zephyr_class, notice.instance, is_personal))
return
if notice.format.startswith("Zephyr error: See") or notice.format.endswith("@(@color(blue))"):
logger.debug("Skipping message we got from Zulip!")
@ -401,20 +437,27 @@ def process_notice(notice: "zephyr.ZNotice", log: Optional[IO[str]]) -> None:
if body.startswith("CC:"):
is_huddle = True
# Map "CC: user1 user2" => "user1@mit.edu, user2@mit.edu"
huddle_recipients = [to_zulip_username(x.strip()) for x in
body.split("\n")[0][4:].split()]
huddle_recipients = [
to_zulip_username(x.strip()) for x in body.split("\n")[0][4:].split()
]
if notice.sender not in huddle_recipients:
huddle_recipients.append(to_zulip_username(notice.sender))
body = body.split("\n", 1)[1]
if options.forward_class_messages and notice.opcode is not None and notice.opcode.lower() == "crypt":
if (
options.forward_class_messages
and notice.opcode is not None
and notice.opcode.lower() == "crypt"
):
body = decrypt_zephyr(zephyr_class, notice.instance.lower(), body)
zeph: ZephyrDict
zeph = {'time': str(notice.time),
'sender': notice.sender,
'zsig': zsig, # logged here but not used by app
'content': body}
zeph = {
'time': str(notice.time),
'sender': notice.sender,
'zsig': zsig, # logged here but not used by app
'content': body,
}
if is_huddle:
zeph['type'] = 'private'
zeph['recipient'] = huddle_recipients
@ -442,8 +485,9 @@ def process_notice(notice: "zephyr.ZNotice", log: Optional[IO[str]]) -> None:
heading = ""
zeph["content"] = heading + zeph["content"]
logger.info("Received a message on %s/%s from %s..." %
(zephyr_class, notice.instance, notice.sender))
logger.info(
"Received a message on %s/%s from %s..." % (zephyr_class, notice.instance, notice.sender)
)
if log is not None:
log.write(json.dumps(zeph) + '\n')
log.flush()
@ -461,11 +505,13 @@ def process_notice(notice: "zephyr.ZNotice", log: Optional[IO[str]]) -> None:
finally:
os._exit(0)
def quit_failed_initialization(message: str) -> str:
logger.error(message)
maybe_kill_child()
sys.exit(1)
def zephyr_init_autoretry() -> None:
backoff = zulip.RandomExponentialBackoff()
while backoff.keep_going():
@ -481,6 +527,7 @@ def zephyr_init_autoretry() -> None:
quit_failed_initialization("Could not initialize Zephyr library, quitting!")
def zephyr_load_session_autoretry(session_path: str) -> None:
backoff = zulip.RandomExponentialBackoff()
while backoff.keep_going():
@ -497,6 +544,7 @@ def zephyr_load_session_autoretry(session_path: str) -> None:
quit_failed_initialization("Could not load saved Zephyr session, quitting!")
def zephyr_subscribe_autoretry(sub: Tuple[str, str, str]) -> None:
backoff = zulip.RandomExponentialBackoff()
while backoff.keep_going():
@ -512,6 +560,7 @@ def zephyr_subscribe_autoretry(sub: Tuple[str, str, str]) -> None:
quit_failed_initialization("Could not subscribe to personals, quitting!")
def zephyr_to_zulip(options: optparse.Values) -> None:
if options.use_sessions and os.path.exists(options.session_path):
logger.info("Loading old session")
@ -542,9 +591,10 @@ def zephyr_to_zulip(options: optparse.Values) -> None:
zeph["stream"] = zeph["class"]
if "instance" in zeph:
zeph["subject"] = zeph["instance"]
logger.info("sending saved message to %s from %s..." %
(zeph.get('stream', zeph.get('recipient')),
zeph['sender']))
logger.info(
"sending saved message to %s from %s..."
% (zeph.get('stream', zeph.get('recipient')), zeph['sender'])
)
send_zulip(zeph)
except Exception:
logger.exception("Could not send saved zephyr:")
@ -558,55 +608,75 @@ def zephyr_to_zulip(options: optparse.Values) -> None:
else:
process_loop(None)
def send_zephyr(zwrite_args: List[str], content: str) -> Tuple[int, str]:
p = subprocess.Popen(zwrite_args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
p = subprocess.Popen(
zwrite_args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
stdout, stderr = p.communicate(input=content)
if p.returncode:
logger.error("zwrite command '%s' failed with return code %d:" % (
" ".join(zwrite_args), p.returncode,))
logger.error(
"zwrite command '%s' failed with return code %d:"
% (
" ".join(zwrite_args),
p.returncode,
)
)
if stdout:
logger.info("stdout: " + stdout)
elif stderr:
logger.warning("zwrite command '%s' printed the following warning:" % (
" ".join(zwrite_args),))
logger.warning(
"zwrite command '%s' printed the following warning:" % (" ".join(zwrite_args),)
)
if stderr:
logger.warning("stderr: " + stderr)
return (p.returncode, stderr)
def send_authed_zephyr(zwrite_args: List[str], content: str) -> Tuple[int, str]:
return send_zephyr(zwrite_args, content)
def send_unauthed_zephyr(zwrite_args: List[str], content: str) -> Tuple[int, str]:
return send_zephyr(zwrite_args + ["-d"], content)
def zcrypt_encrypt_content(zephyr_class: str, instance: str, content: str) -> Optional[str]:
keypath = parse_crypt_table(zephyr_class, instance)
if keypath is None:
return None
# encrypt the message!
p = subprocess.Popen(["gpg",
"--symmetric",
"--no-options",
"--no-default-keyring",
"--keyring=/dev/null",
"--secret-keyring=/dev/null",
"--batch",
"--quiet",
"--no-use-agent",
"--armor",
"--cipher-algo", "AES",
"--passphrase-file",
keypath],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
p = subprocess.Popen(
[
"gpg",
"--symmetric",
"--no-options",
"--no-default-keyring",
"--keyring=/dev/null",
"--secret-keyring=/dev/null",
"--batch",
"--quiet",
"--no-use-agent",
"--armor",
"--cipher-algo",
"AES",
"--passphrase-file",
keypath,
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
encrypted, _ = p.communicate(input=content)
return encrypted
def forward_to_zephyr(message: Dict[str, Any]) -> None:
# 'Any' can be of any type of text
support_heading = "Hi there! This is an automated message from Zulip."
@ -614,12 +684,20 @@ def forward_to_zephyr(message: Dict[str, Any]) -> None:
Feedback button or at support@zulip.com."""
wrapper = textwrap.TextWrapper(break_long_words=False, break_on_hyphens=False)
wrapped_content = "\n".join("\n".join(wrapper.wrap(line))
for line in message["content"].replace("@", "@@").split("\n"))
wrapped_content = "\n".join(
"\n".join(wrapper.wrap(line)) for line in message["content"].replace("@", "@@").split("\n")
)
zwrite_args = ["zwrite", "-n", "-s", message["sender_full_name"],
"-F", "Zephyr error: See http://zephyr.1ts.org/wiki/df",
"-x", "UTF-8"]
zwrite_args = [
"zwrite",
"-n",
"-s",
message["sender_full_name"],
"-F",
"Zephyr error: See http://zephyr.1ts.org/wiki/df",
"-x",
"UTF-8",
]
# Hack to make ctl's fake username setup work :)
if message['type'] == "stream" and zulip_account_email == "ctl@mit.edu":
@ -634,9 +712,8 @@ Feedback button or at support@zulip.com."""
# Forward messages sent to '(instance "WHITESPACE")' back to the
# appropriate WHITESPACE instance for bidirectional mirroring
instance = match_whitespace_instance.group(1)
elif (
instance == "instance %s" % (zephyr_class,)
or instance == "test instance %s" % (zephyr_class,)
elif instance == "instance %s" % (zephyr_class,) or instance == "test instance %s" % (
zephyr_class,
):
# Forward messages to e.g. -c -i white-magic back from the
# place we forward them to
@ -663,15 +740,18 @@ Feedback button or at support@zulip.com."""
zwrite_args.extend(["-C"])
# We drop the @ATHENA.MIT.EDU here because otherwise the
# "CC: user1 user2 ..." output will be unnecessarily verbose.
recipients = [to_zephyr_username(user["email"]).replace("@ATHENA.MIT.EDU", "")
for user in message["display_recipient"]]
recipients = [
to_zephyr_username(user["email"]).replace("@ATHENA.MIT.EDU", "")
for user in message["display_recipient"]
]
logger.info("Forwarding message to %s" % (recipients,))
zwrite_args.extend(recipients)
if message.get("invite_only_stream"):
result = zcrypt_encrypt_content(zephyr_class, instance, wrapped_content)
if result is None:
send_error_zulip("""%s
send_error_zulip(
"""%s
Your Zulip-Zephyr mirror bot was unable to forward that last message \
from Zulip to Zephyr because you were sending to a zcrypted Zephyr \
@ -679,7 +759,9 @@ class and your mirroring bot does not have access to the relevant \
key (perhaps because your AFS tokens expired). That means that while \
Zulip users (like you) received it, Zephyr users did not.
%s""" % (support_heading, support_closing))
%s"""
% (support_heading, support_closing)
)
return
# Proceed with sending a zcrypted message
@ -687,22 +769,24 @@ Zulip users (like you) received it, Zephyr users did not.
zwrite_args.extend(["-O", "crypt"])
if options.test_mode:
logger.debug("Would have forwarded: %s\n%s" %
(zwrite_args, wrapped_content))
logger.debug("Would have forwarded: %s\n%s" % (zwrite_args, wrapped_content))
return
(code, stderr) = send_authed_zephyr(zwrite_args, wrapped_content)
if code == 0 and stderr == "":
return
elif code == 0:
send_error_zulip("""%s
send_error_zulip(
"""%s
Your last message was successfully mirrored to zephyr, but zwrite \
returned the following warning:
%s
%s""" % (support_heading, stderr, support_closing))
%s"""
% (support_heading, stderr, support_closing)
)
return
elif code != 0 and (
stderr.startswith("zwrite: Ticket expired while sending notice to ")
@ -714,7 +798,8 @@ returned the following warning:
if code == 0:
if options.ignore_expired_tickets:
return
send_error_zulip("""%s
send_error_zulip(
"""%s
Your last message was forwarded from Zulip to Zephyr unauthenticated, \
because your Kerberos tickets have expired. It was sent successfully, \
@ -722,13 +807,16 @@ but please renew your Kerberos tickets in the screen session where you \
are running the Zulip-Zephyr mirroring bot, so we can send \
authenticated Zephyr messages for you again.
%s""" % (support_heading, support_closing))
%s"""
% (support_heading, support_closing)
)
return
# zwrite failed and it wasn't because of expired tickets: This is
# probably because the recipient isn't subscribed to personals,
# but regardless, we should just notify the user.
send_error_zulip("""%s
send_error_zulip(
"""%s
Your Zulip-Zephyr mirror bot was unable to forward that last message \
from Zulip to Zephyr. That means that while Zulip users (like you) \
@ -736,20 +824,22 @@ received it, Zephyr users did not. The error message from zwrite was:
%s
%s""" % (support_heading, stderr, support_closing))
%s"""
% (support_heading, stderr, support_closing)
)
return
def maybe_forward_to_zephyr(message: Dict[str, Any]) -> None:
# The key string can be used to direct any type of text.
if (message["sender_email"] == zulip_account_email):
if message["sender_email"] == zulip_account_email:
if not (
(message["type"] == "stream")
or (
message["type"] == "private"
and False
not in [
u["email"].lower().endswith("mit.edu")
for u in message["display_recipient"]
u["email"].lower().endswith("mit.edu") for u in message["display_recipient"]
]
)
):
@ -758,8 +848,9 @@ def maybe_forward_to_zephyr(message: Dict[str, Any]) -> None:
return
timestamp_now = int(time.time())
if float(message["timestamp"]) < timestamp_now - 15:
logger.warning("Skipping out of order message: %s < %s" %
(message["timestamp"], timestamp_now))
logger.warning(
"Skipping out of order message: %s < %s" % (message["timestamp"], timestamp_now)
)
return
try:
forward_to_zephyr(message)
@ -768,6 +859,7 @@ def maybe_forward_to_zephyr(message: Dict[str, Any]) -> None:
# whole process
logger.exception("Error forwarding message:")
def zulip_to_zephyr(options: optparse.Values) -> NoReturn:
# Sync messages from zulip to zephyr
logger.info("Starting syncing messages.")
@ -779,6 +871,7 @@ def zulip_to_zephyr(options: optparse.Values) -> NoReturn:
logger.exception("Error syncing messages:")
backoff.fail()
def subscribed_to_mail_messages() -> bool:
# In case we have lost our AFS tokens and those won't be able to
# parse the Zephyr subs file, first try reading in result of this
@ -787,12 +880,13 @@ def subscribed_to_mail_messages() -> bool:
if stored_result is not None:
return stored_result == "True"
for (cls, instance, recipient) in parse_zephyr_subs(verbose=False):
if (cls.lower() == "mail" and instance.lower() == "inbox"):
if cls.lower() == "mail" and instance.lower() == "inbox":
os.environ["HUMBUG_FORWARD_MAIL_ZEPHYRS"] = "True"
return True
os.environ["HUMBUG_FORWARD_MAIL_ZEPHYRS"] = "False"
return False
def add_zulip_subscriptions(verbose: bool) -> None:
zephyr_subscriptions = set()
skipped = set()
@ -805,7 +899,14 @@ def add_zulip_subscriptions(verbose: bool) -> None:
# We don't support subscribing to (message, *)
if instance == "*":
if recipient == "*":
skipped.add((cls, instance, recipient, "subscribing to all of class message is not supported."))
skipped.add(
(
cls,
instance,
recipient,
"subscribing to all of class message is not supported.",
)
)
continue
# If you're on -i white-magic on zephyr, get on stream white-magic on zulip
# instead of subscribing to stream "message" on zulip
@ -826,8 +927,10 @@ def add_zulip_subscriptions(verbose: bool) -> None:
zephyr_subscriptions.add(cls)
if len(zephyr_subscriptions) != 0:
res = zulip_client.add_subscriptions(list({"name": stream} for stream in zephyr_subscriptions),
authorization_errors_fatal=False)
res = zulip_client.add_subscriptions(
list({"name": stream} for stream in zephyr_subscriptions),
authorization_errors_fatal=False,
)
if res.get("result") != "success":
logger.error("Error subscribing to streams:\n%s" % (res["msg"],))
return
@ -839,9 +942,15 @@ def add_zulip_subscriptions(verbose: bool) -> None:
if already is not None and len(already) > 0:
logger.info("\nAlready subscribed to: %s" % (", ".join(list(already.values())[0]),))
if new is not None and len(new) > 0:
logger.info("\nSuccessfully subscribed to: %s" % (", ".join(list(new.values())[0]),))
logger.info(
"\nSuccessfully subscribed to: %s" % (", ".join(list(new.values())[0]),)
)
if unauthorized is not None and len(unauthorized) > 0:
logger.info("\n" + "\n".join(textwrap.wrap("""\
logger.info(
"\n"
+ "\n".join(
textwrap.wrap(
"""\
The following streams you have NOT been subscribed to,
because they have been configured in Zulip as invitation-only streams.
This was done at the request of users of these Zephyr classes, usually
@ -850,11 +959,19 @@ via zcrypt (in Zulip, we achieve the same privacy goals through invitation-only
If you wish to read these streams in Zulip, you need to contact the people who are
on these streams and already use Zulip. They can subscribe you to them via the
"streams" page in the Zulip web interface:
""")) + "\n\n %s" % (", ".join(unauthorized),))
"""
)
)
+ "\n\n %s" % (", ".join(unauthorized),)
)
if len(skipped) > 0:
if verbose:
logger.info("\n" + "\n".join(textwrap.wrap("""\
logger.info(
"\n"
+ "\n".join(
textwrap.wrap(
"""\
You have some lines in ~/.zephyr.subs that could not be
synced to your Zulip subscriptions because they do not
use "*" as both the instance and recipient and not one of
@ -863,7 +980,11 @@ Zulip has a mechanism for forwarding. Zulip does not
allow subscribing to only some subjects on a Zulip
stream, so this tool has not created a corresponding
Zulip subscription to these lines in ~/.zephyr.subs:
""")) + "\n")
"""
)
)
+ "\n"
)
for (cls, instance, recipient, reason) in skipped:
if verbose:
@ -873,15 +994,25 @@ Zulip subscription to these lines in ~/.zephyr.subs:
logger.info(" [%s,%s,%s]" % (cls, instance, recipient))
if len(skipped) > 0:
if verbose:
logger.info("\n" + "\n".join(textwrap.wrap("""\
logger.info(
"\n"
+ "\n".join(
textwrap.wrap(
"""\
If you wish to be subscribed to any Zulip streams related
to these .zephyrs.subs lines, please do so via the Zulip
web interface.
""")) + "\n")
"""
)
)
+ "\n"
)
def valid_stream_name(name: str) -> bool:
return name != ""
def parse_zephyr_subs(verbose: bool = False) -> Set[Tuple[str, str, str]]:
zephyr_subscriptions = set() # type: Set[Tuple[str, str, str]]
subs_file = os.path.join(os.environ["HOME"], ".zephyr.subs")
@ -910,6 +1041,7 @@ def parse_zephyr_subs(verbose: bool = False) -> Set[Tuple[str, str, str]]:
zephyr_subscriptions.add((cls.strip(), instance.strip(), recipient.strip()))
return zephyr_subscriptions
def open_logger() -> logging.Logger:
if options.log_path is not None:
log_file = options.log_path
@ -919,8 +1051,7 @@ def open_logger() -> logging.Logger:
else:
log_file = "/var/log/zulip/mirror-log"
else:
f = tempfile.NamedTemporaryFile(prefix="zulip-log.%s." % (options.user,),
delete=False)
f = tempfile.NamedTemporaryFile(prefix="zulip-log.%s." % (options.user,), delete=False)
log_file = f.name
# Close the file descriptor, since the logging system will
# reopen it anyway.
@ -935,6 +1066,7 @@ def open_logger() -> logging.Logger:
logger.addHandler(file_handler)
return logger
def configure_logger(logger: logging.Logger, direction_name: Optional[str]) -> None:
if direction_name is None:
log_format = "%(message)s"
@ -949,89 +1081,70 @@ def configure_logger(logger: logging.Logger, direction_name: Optional[str]) -> N
for handler in root_logger.handlers:
handler.setFormatter(formatter)
def parse_args() -> Tuple[optparse.Values, List[str]]:
parser = optparse.OptionParser()
parser.add_option('--forward-class-messages',
default=False,
help=optparse.SUPPRESS_HELP,
action='store_true')
parser.add_option('--shard',
help=optparse.SUPPRESS_HELP)
parser.add_option('--noshard',
default=False,
help=optparse.SUPPRESS_HELP,
action='store_true')
parser.add_option('--resend-log',
dest='logs_to_resend',
help=optparse.SUPPRESS_HELP)
parser.add_option('--enable-resend-log',
dest='resend_log_path',
help=optparse.SUPPRESS_HELP)
parser.add_option('--log-path',
dest='log_path',
help=optparse.SUPPRESS_HELP)
parser.add_option('--stream-file-path',
dest='stream_file_path',
default="/home/zulip/public_streams",
help=optparse.SUPPRESS_HELP)
parser.add_option('--no-forward-personals',
dest='forward_personals',
help=optparse.SUPPRESS_HELP,
default=True,
action='store_false')
parser.add_option('--forward-mail-zephyrs',
dest='forward_mail_zephyrs',
help=optparse.SUPPRESS_HELP,
default=False,
action='store_true')
parser.add_option('--no-forward-from-zulip',
default=True,
dest='forward_from_zulip',
help=optparse.SUPPRESS_HELP,
action='store_false')
parser.add_option('--verbose',
default=False,
help=optparse.SUPPRESS_HELP,
action='store_true')
parser.add_option('--sync-subscriptions',
default=False,
action='store_true')
parser.add_option('--ignore-expired-tickets',
default=False,
action='store_true')
parser.add_option('--site',
default=DEFAULT_SITE,
help=optparse.SUPPRESS_HELP)
parser.add_option('--on-startup-command',
default=None,
help=optparse.SUPPRESS_HELP)
parser.add_option('--user',
default=os.environ["USER"],
help=optparse.SUPPRESS_HELP)
parser.add_option('--stamp-path',
default="/afs/athena.mit.edu/user/t/a/tabbott/for_friends",
help=optparse.SUPPRESS_HELP)
parser.add_option('--session-path',
default=None,
help=optparse.SUPPRESS_HELP)
parser.add_option('--nagios-class',
default=None,
help=optparse.SUPPRESS_HELP)
parser.add_option('--nagios-path',
default=None,
help=optparse.SUPPRESS_HELP)
parser.add_option('--use-sessions',
default=False,
action='store_true',
help=optparse.SUPPRESS_HELP)
parser.add_option('--test-mode',
default=False,
help=optparse.SUPPRESS_HELP,
action='store_true')
parser.add_option('--api-key-file',
default=os.path.join(os.environ["HOME"], "Private", ".humbug-api-key"))
parser.add_option(
'--forward-class-messages', default=False, help=optparse.SUPPRESS_HELP, action='store_true'
)
parser.add_option('--shard', help=optparse.SUPPRESS_HELP)
parser.add_option('--noshard', default=False, help=optparse.SUPPRESS_HELP, action='store_true')
parser.add_option('--resend-log', dest='logs_to_resend', help=optparse.SUPPRESS_HELP)
parser.add_option('--enable-resend-log', dest='resend_log_path', help=optparse.SUPPRESS_HELP)
parser.add_option('--log-path', dest='log_path', help=optparse.SUPPRESS_HELP)
parser.add_option(
'--stream-file-path',
dest='stream_file_path',
default="/home/zulip/public_streams",
help=optparse.SUPPRESS_HELP,
)
parser.add_option(
'--no-forward-personals',
dest='forward_personals',
help=optparse.SUPPRESS_HELP,
default=True,
action='store_false',
)
parser.add_option(
'--forward-mail-zephyrs',
dest='forward_mail_zephyrs',
help=optparse.SUPPRESS_HELP,
default=False,
action='store_true',
)
parser.add_option(
'--no-forward-from-zulip',
default=True,
dest='forward_from_zulip',
help=optparse.SUPPRESS_HELP,
action='store_false',
)
parser.add_option('--verbose', default=False, help=optparse.SUPPRESS_HELP, action='store_true')
parser.add_option('--sync-subscriptions', default=False, action='store_true')
parser.add_option('--ignore-expired-tickets', default=False, action='store_true')
parser.add_option('--site', default=DEFAULT_SITE, help=optparse.SUPPRESS_HELP)
parser.add_option('--on-startup-command', default=None, help=optparse.SUPPRESS_HELP)
parser.add_option('--user', default=os.environ["USER"], help=optparse.SUPPRESS_HELP)
parser.add_option(
'--stamp-path',
default="/afs/athena.mit.edu/user/t/a/tabbott/for_friends",
help=optparse.SUPPRESS_HELP,
)
parser.add_option('--session-path', default=None, help=optparse.SUPPRESS_HELP)
parser.add_option('--nagios-class', default=None, help=optparse.SUPPRESS_HELP)
parser.add_option('--nagios-path', default=None, help=optparse.SUPPRESS_HELP)
parser.add_option(
'--use-sessions', default=False, action='store_true', help=optparse.SUPPRESS_HELP
)
parser.add_option(
'--test-mode', default=False, help=optparse.SUPPRESS_HELP, action='store_true'
)
parser.add_option(
'--api-key-file', default=os.path.join(os.environ["HOME"], "Private", ".humbug-api-key")
)
return parser.parse_args()
def die_gracefully(signal: int, frame: FrameType) -> None:
if CURRENT_STATE == States.ZulipToZephyr or CURRENT_STATE == States.ChildSending:
# this is a child process, so we want os._exit (no clean-up necessary)
@ -1047,6 +1160,7 @@ def die_gracefully(signal: int, frame: FrameType) -> None:
sys.exit(1)
if __name__ == "__main__":
# Set the SIGCHLD handler back to SIG_DFL to prevent these errors
# when importing the "requests" module after being restarted using
@ -1070,10 +1184,18 @@ if __name__ == "__main__":
api_key = os.environ.get("HUMBUG_API_KEY")
else:
if not os.path.exists(options.api_key_file):
logger.error("\n" + "\n".join(textwrap.wrap("""\
logger.error(
"\n"
+ "\n".join(
textwrap.wrap(
"""\
Could not find API key file.
You need to either place your api key file at %s,
or specify the --api-key-file option.""" % (options.api_key_file,))))
or specify the --api-key-file option."""
% (options.api_key_file,)
)
)
)
sys.exit(1)
api_key = open(options.api_key_file).read().strip()
# Store the API key in the environment so that our children
@ -1086,12 +1208,14 @@ or specify the --api-key-file option.""" % (options.api_key_file,))))
zulip_account_email = options.user + "@mit.edu"
import zulip
zulip_client = zulip.Client(
email=zulip_account_email,
api_key=api_key,
verbose=True,
client="zephyr_mirror",
site=options.site)
site=options.site,
)
start_time = time.time()
@ -1110,9 +1234,11 @@ or specify the --api-key-file option.""" % (options.api_key_file,))))
elif options.user is not None:
# Personals mirror on behalf of another user.
pgrep_query = "%s.*--user=%s" % (pgrep_query, options.user)
proc = subprocess.Popen(['pgrep', '-U', os.environ["USER"], "-f", pgrep_query],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc = subprocess.Popen(
['pgrep', '-U', os.environ["USER"], "-f", pgrep_query],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, _err_unused = proc.communicate()
for pid in map(int, out.split()):
if pid == os.getpid() or pid == os.getppid():
@ -1149,6 +1275,7 @@ or specify the --api-key-file option.""" % (options.api_key_file,))))
CURRENT_STATE = States.ZephyrToZulip
import zephyr
logger_name = "zephyr=>zulip"
if options.shard is not None:
logger_name += "(%s)" % (options.shard,)

25
zulip/setup.py
View file

@ -8,20 +8,24 @@ from typing import Any, Dict, Generator, List, Tuple
with open("README.md") as fh:
long_description = fh.read()
def version() -> str:
version_py = os.path.join(os.path.dirname(__file__), "zulip", "__init__.py")
with open(version_py) as in_handle:
version_line = next(itertools.dropwhile(lambda x: not x.startswith("__version__"),
in_handle))
version_line = next(
itertools.dropwhile(lambda x: not x.startswith("__version__"), in_handle)
)
version = version_line.split('=')[-1].strip().replace('"', '')
return version
def recur_expand(target_root: Any, dir: Any) -> Generator[Tuple[str, List[str]], None, None]:
for root, _, files in os.walk(dir):
paths = [os.path.join(root, f) for f in files]
if len(paths):
yield os.path.join(target_root, root), paths
# We should be installable with either setuptools or distutils.
package_info = dict(
name='zulip',
@ -56,22 +60,24 @@ package_info = dict(
'zulip-send=zulip.send:main',
'zulip-api-examples=zulip.api_examples:main',
'zulip-matrix-bridge=integrations.bridge_with_matrix.matrix_bridge:main',
'zulip-api=zulip.cli:cli'
'zulip-api=zulip.cli:cli',
],
},
package_data={'zulip': ["py.typed"]},
) # type: Dict[str, Any]
setuptools_info = dict(
install_requires=['requests[security]>=0.12.1',
'matrix_client',
'distro',
'click',
],
install_requires=[
'requests[security]>=0.12.1',
'matrix_client',
'distro',
'click',
],
)
try:
from setuptools import find_packages, setup
package_info.update(setuptools_info)
package_info['packages'] = find_packages(exclude=['tests'])
@ -82,7 +88,8 @@ except ImportError:
# Manual dependency check
try:
import requests
assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
assert LooseVersion(requests.__version__) >= LooseVersion('0.12.1')
except (ImportError, AssertionError):
print("requests >=0.12.1 is not installed", file=sys.stderr)
sys.exit(1)

17
zulip/tests/test_default_arguments.py
View file

@ -12,7 +12,6 @@ from zulip import ZulipError
class TestDefaultArguments(TestCase):
def test_invalid_arguments(self) -> None:
parser = zulip.add_default_arguments(argparse.ArgumentParser(usage="lorem ipsum"))
with self.assertRaises(SystemExit) as cm:
@ -20,14 +19,18 @@ class TestDefaultArguments(TestCase):
parser.parse_args(['invalid argument'])
self.assertEqual(cm.exception.code, 2)
# Assert that invalid arguments exit with printing the full usage (non-standard behavior)
self.assertTrue(mock_stderr.getvalue().startswith("""usage: lorem ipsum
self.assertTrue(
mock_stderr.getvalue().startswith(
"""usage: lorem ipsum
optional arguments:
-h, --help show this help message and exit
Zulip API configuration:
--site ZULIP_SITE Zulip server URI
"""))
"""
)
)
@patch('os.path.exists', return_value=False)
def test_config_path_with_tilde(self, mock_os_path_exists: bool) -> None:
@ -37,8 +40,12 @@ Zulip API configuration:
with self.assertRaises(ZulipError) as cm:
zulip.init_from_options(args)
expanded_test_path = os.path.abspath(os.path.expanduser(test_path))
self.assertEqual(str(cm.exception), 'api_key or email not specified and '
'file {} does not exist'.format(expanded_test_path))
self.assertEqual(
str(cm.exception),
'api_key or email not specified and '
'file {} does not exist'.format(expanded_test_path),
)
if __name__ == '__main__':
unittest.main()

1
zulip/tests/test_hash_util_decode.py
View file

@ -20,5 +20,6 @@ class TestHashUtilDecode(TestCase):
with self.subTest(encoded_string=encoded_string):
self.assertEqual(zulip.hash_util_decode(encoded_string), decoded_string)
if __name__ == '__main__':
unittest.main()

1023
zulip/zulip/__init__.py
View file

File diff suppressed because it is too large Load diff

16
zulip/zulip/api_examples.py
View file

@ -10,19 +10,21 @@ def main() -> None:
Prints the path to the Zulip API example scripts."""
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument('script_name',
nargs='?',
default='',
help='print path to the script <script_name>')
parser.add_argument(
'script_name', nargs='?', default='', help='print path to the script <script_name>'
)
args = parser.parse_args()
zulip_path = os.path.abspath(os.path.dirname(zulip.__file__))
examples_path = os.path.abspath(os.path.join(zulip_path, 'examples', args.script_name))
if os.path.isdir(examples_path) or (args.script_name and os.path.isfile(examples_path)):
print(examples_path)
else:
raise OSError("Examples cannot be accessed at {}: {} does not exist!"
.format(examples_path,
"File" if args.script_name else "Directory"))
raise OSError(
"Examples cannot be accessed at {}: {} does not exist!".format(
examples_path, "File" if args.script_name else "Directory"
)
)
if __name__ == '__main__':
main()

16
zulip/zulip/examples/create-user
View file

@ -23,9 +23,13 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
print(client.create_user({
'email': options.new_email,
'password': options.new_password,
'full_name': options.new_full_name,
'short_name': options.new_short_name
}))
print(
client.create_user(
{
'email': options.new_email,
'password': options.new_password,
'full_name': options.new_full_name,
'short_name': options.new_short_name,
}
)
)

20
zulip/zulip/examples/edit-stream
View file

@ -29,11 +29,15 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
print(client.update_stream({
'stream_id': options.stream_id,
'description': quote(options.description),
'new_name': quote(options.new_name),
'is_private': options.private,
'is_announcement_only': options.announcement_only,
'history_public_to_subscribers': options.history_public_to_subscribers
}))
print(
client.update_stream(
{
'stream_id': options.stream_id,
'description': quote(options.description),
'new_name': quote(options.new_name),
'is_private': options.private,
'is_announcement_only': options.announcement_only,
'history_public_to_subscribers': options.history_public_to_subscribers,
}
)
)

10
zulip/zulip/examples/get-history
View file

@ -15,8 +15,12 @@ Example: get-history --stream announce --topic important"""
parser = zulip.add_default_arguments(argparse.ArgumentParser(usage=usage))
parser.add_argument('--stream', required=True, help="The stream name to get the history")
parser.add_argument('--topic', help="The topic name to get the history")
parser.add_argument('--filename', default='history.json', help="The file name to store the fetched \
history.\n Default 'history.json'")
parser.add_argument(
'--filename',
default='history.json',
help="The file name to store the fetched \
history.\n Default 'history.json'",
)
options = parser.parse_args()
client = zulip.init_from_options(options)
@ -33,7 +37,7 @@ request = {
'num_after': 1000,
'narrow': narrow,
'client_gravatar': False,
'apply_markdown': False
'apply_markdown': False,
}
all_messages = [] # type: List[Dict[str, Any]]

22
zulip/zulip/examples/get-messages
View file

@ -28,12 +28,16 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
print(client.get_messages({
'anchor': options.anchor,
'use_first_unread_anchor': options.use_first_unread_anchor,
'num_before': options.num_before,
'num_after': options.num_after,
'narrow': options.narrow,
'client_gravatar': options.client_gravatar,
'apply_markdown': options.apply_markdown
}))
print(
client.get_messages(
{
'anchor': options.anchor,
'use_first_unread_anchor': options.use_first_unread_anchor,
'num_before': options.num_before,
'num_after': options.num_after,
'narrow': options.narrow,
'client_gravatar': options.client_gravatar,
'apply_markdown': options.apply_markdown,
}
)
)

15
zulip/zulip/examples/mute-topic
View file

@ -18,13 +18,10 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
OPERATIONS = {
'mute': 'add',
'unmute': 'remove'
}
OPERATIONS = {'mute': 'add', 'unmute': 'remove'}
print(client.mute_topic({
'op': OPERATIONS[options.op],
'stream': options.stream,
'topic': options.topic
}))
print(
client.mute_topic(
{'op': OPERATIONS[options.op], 'stream': options.stream, 'topic': options.topic}
)
)

2
zulip/zulip/examples/print-events
View file

@ -19,9 +19,11 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
def print_event(event: Dict[str, Any]) -> None:
print(event)
# This is a blocking call, and will continuously poll for new events
# Note also the filter here is messages to the stream Denmark; if you
# don't specify event_types it'll print all events.

2
zulip/zulip/examples/print-messages
View file

@ -19,8 +19,10 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
def print_message(message: Dict[str, Any]) -> None:
print(message)
# This is a blocking call, and will continuously poll for new messages
client.call_on_each_message(print_message)

3
zulip/zulip/examples/subscribe
View file

@ -20,5 +20,4 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
print(client.add_subscriptions([{"name": stream_name} for stream_name in
options.streams.split()]))
print(client.add_subscriptions([{"name": stream_name} for stream_name in options.streams.split()]))

10
zulip/zulip/examples/update-message-flags
View file

@ -19,8 +19,8 @@ options = parser.parse_args()
client = zulip.init_from_options(options)
print(client.update_message_flags({
'op': options.op,
'flag': options.flag,
'messages': options.messages
}))
print(
client.update_message_flags(
{'op': options.op, 'flag': options.flag, 'messages': options.messages}
)
)

1
zulip/zulip/examples/upload-file
View file

@ -10,6 +10,7 @@ import zulip
class StringIO(_StringIO):
name = '' # https://github.com/python/typeshed/issues/598
usage = """upload-file [options]
Upload a file, and print the corresponding URI.

16
zulip/zulip/examples/welcome-message
View file

@ -52,13 +52,16 @@ streams_to_watch = ['new members']
# These streams will cause anyone who sends a message there to be removed from the watchlist
streams_to_cancel = ['development help']
def get_watchlist() -> List[Any]:
storage = client.get_storage()
return list(storage['storage'].values())
def set_watchlist(watchlist: List[str]) -> None:
client.update_storage({'storage': dict(enumerate(watchlist))})
def handle_event(event: Dict[str, Any]) -> None:
try:
if event['type'] == 'realm_user' and event['op'] == 'add':
@ -74,11 +77,13 @@ def handle_event(event: Dict[str, Any]) -> None:
if event['message']['sender_email'] in watchlist:
watchlist.remove(event['message']['sender_email'])
if stream not in streams_to_cancel:
client.send_message({
'type': 'private',
'to': event['message']['sender_email'],
'content': welcome_text.format(event['message']['sender_short_name'])
})
client.send_message(
{
'type': 'private',
'to': event['message']['sender_email'],
'content': welcome_text.format(event['message']['sender_short_name']),
}
)
set_watchlist(watchlist)
return
except Exception as err:
@ -89,5 +94,6 @@ def start_event_handler() -> None:
print("Starting event handler...")
client.call_on_each_event(handle_event, event_types=['realm_user', 'message'])
client = zulip.Client()
start_event_handler()

42
zulip/zulip/send.py
View file

@ -12,12 +12,15 @@ logging.basicConfig()
log = logging.getLogger('zulip-send')
def do_send_message(client: zulip.Client, message_data: Dict[str, Any]) -> bool:
'''Sends a message and optionally prints status about the same.'''
if message_data['type'] == 'stream':
log.info('Sending message to stream "%s", subject "%s"... ' %
(message_data['to'], message_data['subject']))
log.info(
'Sending message to stream "%s", subject "%s"... '
% (message_data['to'], message_data['subject'])
)
else:
log.info('Sending message to %s... ' % (message_data['to'],))
response = client.send_message(message_data)
@ -28,6 +31,7 @@ def do_send_message(client: zulip.Client, message_data: Dict[str, Any]) -> bool:
log.error(response['msg'])
return False
def main() -> int:
usage = """zulip-send [options] [recipient...]
@ -41,22 +45,29 @@ def main() -> int:
parser = zulip.add_default_arguments(argparse.ArgumentParser(usage=usage))
parser.add_argument('recipients',
nargs='*',
help='email addresses of the recipients of the message')
parser.add_argument(
'recipients', nargs='*', help='email addresses of the recipients of the message'
)
parser.add_argument('-m', '--message',
help='Specifies the message to send, prevents interactive prompting.')
parser.add_argument(
'-m', '--message', help='Specifies the message to send, prevents interactive prompting.'
)
group = parser.add_argument_group('Stream parameters')
group.add_argument('-s', '--stream',
dest='stream',
action='store',
help='Allows the user to specify a stream for the message.')
group.add_argument('-S', '--subject',
dest='subject',
action='store',
help='Allows the user to specify a subject for the message.')
group.add_argument(
'-s',
'--stream',
dest='stream',
action='store',
help='Allows the user to specify a stream for the message.',
)
group.add_argument(
'-S',
'--subject',
dest='subject',
action='store',
help='Allows the user to specify a subject for the message.',
)
options = parser.parse_args()
@ -93,5 +104,6 @@ def main() -> int:
return 1
return 0
if __name__ == '__main__':
sys.exit(main())