diff --git a/zulip/integrations/slack/slackdata2zulipdata.py b/zulip/integrations/slack/slackdata2zulipdata.py new file mode 100755 index 0000000..1c0c34f --- /dev/null +++ b/zulip/integrations/slack/slackdata2zulipdata.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +import os +import json +import hashlib +import sys +import argparse +import shutil +import subprocess +import zipfile + + +# Transported from https://github.com/zulip/zulip/blob/master/zerver/lib/export.py +def rm_tree(path): + # type: (str) -> None + if os.path.exists(path): + shutil.rmtree(path) + +def users2zerver_userprofile(slack_dir, realm_id, timestamp, domain_name): + # type: () -> None + print('######### IMPORTING USERS STARTED #########\n') + users = json.load(open(slack_dir + '/users.json')) + zerver_userprofile = [] + added_users = {} + user_id_count = 1 + for user in users: + slack_user_id = user['id'] + profile = user['profile'] + DESKTOP_NOTIFICATION = True + if 'email' not in profile: + email = (hashlib.blake2b(user['real_name'].encode()).hexdigest() + + "@%s" % (domain_name)) + else: + email = profile['email'] + + # userprofile's quota is hardcoded as per + # https://github.com/zulip/zulip/blob/e1498988d9094961e6f9988fb308b3e7310a8e74/zerver/migrations/0059_userprofile_quota.py#L18 + userprofile = dict( + enable_desktop_notifications=DESKTOP_NOTIFICATION, + is_staff=user.get('is_admin', False), + # avatar_source='G', + is_bot=user.get('is_bot', False), + avatar_version=1, + autoscroll_forever=False, + default_desktop_notifications=True, + timezone=user.get("tz", ""), + default_sending_stream=None, + enable_offline_email_notifications=True, + user_permissions=[], # This is Zulip-specific + is_mirror_dummy=False, + pointer=-1, + default_events_register_stream=None, + is_realm_admin=user.get('is_owner', False), + invites_granted=0, + enter_sends=True, + bot_type=1 if user.get('is_bot', False) else None, + enable_stream_sounds=False, + is_api_super_user=False, + rate_limits="", + last_login=timestamp, + tos_version=None, + default_all_public_streams=False, + full_name=user.get('real_name', user['name']), + twenty_four_hour_time=False, + groups=[], # This is Zulip-specific + muted_topics=[], + enable_online_push_notifications=False, + alert_words="[]", + # bot_owner=None, # This is Zulip-specific + short_name=user['name'], + enable_offline_push_notifications=True, + left_side_userlist=False, + enable_stream_desktop_notifications=False, + enable_digest_emails=True, + last_pointer_updater="", + email=email, + date_joined=timestamp, + last_reminder=timestamp, + is_superuser=False, + tutorial_status="T", + default_language="en", + enable_sounds=True, + pm_content_in_desktop_notifications=True, + is_active=user['deleted'], + onboarding_steps="[]", + emojiset="google", + emoji_alt_code=False, + realm=realm_id, + quota=1073741824, + invites_used=0, + id=user_id_count) + + # TODO map the avatar + # zerver auto-infer the url from Gravatar instead of from a specified + # url; zerver.lib.avatar needs to be patched + # profile['image_32'], Slack has 24, 32, 48, 72, 192, 512 size range + + zerver_userprofile.append(userprofile) + added_users[slack_user_id] = user_id_count + user_id_count += 1 + print(u"{} -> {}\nCreated\n".format(user['name'], userprofile['email'])) + print('######### IMPORTING USERS FINISHED #########\n') + return zerver_userprofile, added_users + +def channels2zerver_stream(slack_dir, realm_id, added_users): + # type: (Dict[str, Dict[str, str]]) -> None + print('######### IMPORTING CHANNELS STARTED #########\n') + channels = json.load(open(slack_dir + '/channels.json')) + added_channels = {} + zerver_stream = [] + stream_id_count = 1 + zerver_subscription = [] + zerver_recipient = [] + for channel in channels: + # slack_channel_id = channel['id'] + + # map Slack's topic and purpose content into Zulip's stream description. + # WARN This mapping is lossy since the topic.creator, topic.last_set, + # purpose.creator, purpose.last_set fields are not preserved. + description = "topic: {}\npurpose: {}".format(channel["topic"]["value"], + channel["purpose"]["value"]) + + # construct the stream object and append it to zerver_stream + stream = dict( + realm=realm_id, + name=channel["name"], + deactivated=channel["is_archived"], + description=description, + invite_only=not channel["is_general"], + date_created=channel["created"], + id=stream_id_count) + zerver_stream.append(stream) + added_channels[stream['name']] = stream_id_count + + # construct the subscription object and append it to zerver_subscription + for member in channel['members']: + sub = dict( + recipient=added_users[member], + notifications=False, + color="#c2c2c2", + desktop_notifications=True, + pin_to_top=False, + in_home_view=True, + active=True, + user_profile=added_users[member], + id=stream_id_count) # TODO is this the correct interpretation? + zerver_subscription.append(sub) + + # recipient + # type_id's + # 1: private message + # 2: stream + # 3: huddle + # TOODO currently the goal is to map Slack's standard export + # This defaults to 2 + # TOODO do private message subscriptions between each users have to + # be generated from scratch? + rcpt = dict( + type=2, + type_id=stream_id_count, + id=added_users[member]) + zerver_recipient.append(rcpt) + + stream_id_count += 1 + print(u"{} -> created\n".format(channel['name'])) + + # TODO map Slack's pins to Zulip's stars + # There is the security model that Slack's pins are known to the team owner + # as evident from where it is stored at (channels) + # "pins": [ + # { + # "id": "1444755381.000003", + # "type": "C", + # "user": "U061A5N1G", + # "owner": "U061A5N1G", + # "created": "1444755463" + # } + # ], + print('######### IMPORTING STREAMS FINISHED #########\n') + return zerver_stream, added_channels, zerver_subscription, zerver_recipient + +def channelmessage2zerver_message(slack_dir, channel, added_users, added_channels): + json_names = os.listdir(slack_dir + '/' + channel) + zerver_message = [] + msg_id_count = 1 + for json_name in json_names: + msgs = json.load(open(slack_dir + '/%s/%s' % (channel, json_name))) + for msg in msgs: + text = msg['text'] + try: + user = msg.get('user', msg['file']['user']) + except KeyError: + # black magic, explain this later TOODOO + user = msg['user'] + zulip_message = dict( + sending_client=1, + rendered_content_version=1, # This is Zulip-specific + has_image=msg['has_image'], + subject=channel, # This is Zulip-specific + pub_date=msg['ts'], + id=msg_id_count, + has_attachment=False, # attachment will be posted in the subsequent message; this is how Slack does it, less like email + edit_history=None, + sender=added_users[user], # map slack id to zulip id + content=text, # TODO sanitize slack text, which contains <@msg['user']|short_name> + rendered_content=text, # slack doesn't cache this + recipient=added_channels[channel], + last_edit_time=None, + has_link=msg['has_link']) + zerver_message.append(zulip_message) + return zerver_message + +def main(slack_zip_file): + # type: (str) -> None + + slack_dir = slack_zip_file.replace('.zip', '') + subprocess.check_call(['unzip', slack_zip_file]) + # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: + # zip_ref.extractall(slack_dir) + + from datetime import datetime + # TODO fetch realm config from zulip config + DOMAIN_NAME = "zulipchat.com" + REALM_ID = 1 # TODO how to find this + REALM_NAME = "FleshEatingBatswithFangs" + NOW = datetime.utcnow().timestamp() + + script_path = os.path.dirname(os.path.abspath(__file__)) + '/' + zerver_realm_skeleton = json.load(open(script_path + 'zerver_realm_skeleton.json')) + zerver_realm_skeleton[0]['id'] = REALM_ID + zerver_realm_skeleton[0]['string_id'] = 'zulip' # subdomain / short_name of realm + zerver_realm_skeleton[0]['name'] = REALM_NAME + zerver_realm_skeleton[0]['date_created'] = NOW + + # Make sure the directory output is clean + output_dir = 'zulip_data' + rm_tree(output_dir) + os.makedirs(output_dir) + + realm = dict(zerver_defaultstream=[], # TODO + zerver_client=[{"name": "populate_db", "id": 1}, + {"name": "website", "id": 2}, + {"name": "API", "id": 3}], + zerver_userpresence=[], # TODO + zerver_userprofile_mirrordummy=[], + zerver_realmdomain=[{"realm": REALM_ID, + "allow_subdomains": False, + "domain": DOMAIN_NAME, + "id": REALM_ID}], + zerver_useractivity=[], + zerver_realm=zerver_realm_skeleton, + zerver_huddle=[], + zerver_userprofile_crossrealm=[], + zerver_useractivityinterval=[], + zerver_realmfilter=[], + zerver_realmemoji=[]) + + zerver_userprofile, added_users = users2zerver_userprofile(slack_dir, + REALM_ID, + int(NOW), + DOMAIN_NAME) + realm['zerver_userprofile'] = zerver_userprofile + + zerver_stream, added_channels, zerver_subscription, zerver_recipient = channels2zerver_stream(slack_dir, REALM_ID, added_users) + realm['zerver_stream'] = zerver_stream + realm['zerver_subscription'] = zerver_subscription + realm['zerver_recipient'] = zerver_recipient + # IO + json.dump(realm, open(output_dir + '/realm.json', 'w')) + + # now for message.json + message_json = {} + zerver_message = [] + # TODO map zerver_usermessage + for channel in added_channels.keys(): + zerver_message.append(channelmessage2zerver_message(slack_dir, channel, + added_users, added_channels)) + message_json['zerver_message'] = zerver_message + # IO + json.dump(message_json, open(output_dir + '/message.json', 'w')) + + # TODO + # attachments + + # remove slack dir + rm_tree(slack_dir) + + # compress the folder + subprocess.check_call(['zip', '-r', output_dir + '.zip', output_dir]) + + # remove zulip dir + rm_tree(output_dir) + + sys.exit(0) + +if __name__ == '__main__': + # from django.conf import settings + # settings_module = "settings.py" + # os.environ['DJANGO_SETTINGS_MODULE'] = settings_module + description = ("script to convert Slack export data into Zulip export data") + parser = argparse.ArgumentParser(description=description) + slack_zip_file = sys.argv[1] + main(slack_zip_file) diff --git a/zulip/integrations/slack/zerver_realm_skeleton.json b/zulip/integrations/slack/zerver_realm_skeleton.json new file mode 100644 index 0000000..b208cbf --- /dev/null +++ b/zulip/integrations/slack/zerver_realm_skeleton.json @@ -0,0 +1,36 @@ +[{ + "message_retention_days": null, + "inline_image_preview": true, + "name_changes_disabled": false, + "string_id": "zulip", + "icon_version": 1, + "waiting_period_threshold": 0, + "email_changes_disabled": false, + "deactivated": false, + "notifications_stream": null, + "restricted_to_domain": true, + "show_digest_email": true, + "allow_message_editing": true, + "description": "The Zulip development environment default organization. It's great for testing!", + "default_language": "en", + "icon_source": "G", + "invite_required": false, + "invite_by_admins_only": false, + "create_stream_by_admins_only": false, + "mandatory_topics": false, + "inline_url_embed_preview": true, + "message_content_edit_limit_seconds": 600, + "authentication_methods": [ + ["Google", true], + ["Email", true], + ["GitHub", true], + ["LDAP", true], + ["Dev", true], + ["RemoteUser", true] + ], + "name": "", + "org_type": 1, + "add_emoji_by_admins_only": false, + "date_created": null, + "id": 1 +}]