python-zulip-api/zulip/integrations/slack/slackdata2zulipdata.py

#!/usr/bin/env python3
import os
import json
import hashlib
import sys
import argparse
import shutil
import subprocess
import re

import requests

from typing import Any, Dict, List
# stubs
user_profile_stub = Dict[str, Any]
added_users_stub = Dict[str, int]


# Transported from https://github.com/zulip/zulip/blob/master/zerver/lib/export.py
def rm_tree(path: str) -> None:
    if os.path.exists(path):
        shutil.rmtree(path)

def users2zerver_userprofile(slack_dir: str, realm_id: int, timestamp: Any,
                             domain_name: str) -> (List[user_profile_stub], added_users_stub):
    """
    Returns:
    1. zerver_userprofile, which is a list of user profile
    2. added_users, which is a dictionary to map from slack user id to zulip
       user id
    """
    print('######### IMPORTING USERS STARTED #########\n')
    users = json.load(open(slack_dir + '/users.json'))
    zerver_userprofile = []
    added_users = {}
    user_id_count = 1
    for user in users:
        slack_user_id = user['id']
        profile = user['profile']
        DESKTOP_NOTIFICATION = True

        # email
        if 'email' not in profile:
            email = (hashlib.blake2b(user['real_name'].encode()).hexdigest() +
                     "@%s" % (domain_name))
        else:
            email = profile['email']

        # avatar
        # ref: https://chat.zulip.org/help/change-your-avatar
        avatar_source = 'U'
        if 'gravatar.com' in profile['image_32']:
            # use the avatar from gravatar
            avatar_source = 'G'

        # timezone
        _default_timezone = "America/New_York"
        timezone = user.get("tz", _default_timezone)
        if timezone is None or '/' not in timezone:
            timezone = _default_timezone

        if user['deleted'] is False:
            if user['real_name'] == '':
                full_name = user['name']
            else:
                full_name = user['real_name']

        # userprofile's quota is hardcoded as per
        # https://github.com/zulip/zulip/blob/e1498988d9094961e6f9988fb308b3e7310a8e74/zerver/migrations/0059_userprofile_quota.py#L18
        userprofile = dict(
            enable_desktop_notifications=DESKTOP_NOTIFICATION,
            is_staff=user.get('is_admin', False),
            avatar_source=avatar_source,
            is_bot=user.get('is_bot', False),
            avatar_version=1,
            autoscroll_forever=False,
            default_desktop_notifications=True,
            timezone=timezone,
            default_sending_stream=None,
            enable_offline_email_notifications=True,
            user_permissions=[],  # This is Zulip-specific
            is_mirror_dummy=False,
            pointer=-1,
            default_events_register_stream=None,
            is_realm_admin=user.get('is_owner', False),
            # invites_granted=0,  # TODO
            enter_sends=True,
            bot_type=1 if user.get('is_bot', False) else None,
            enable_stream_sounds=False,
            is_api_super_user=False,
            rate_limits="",
            last_login=timestamp,
            tos_version=None,
            default_all_public_streams=False,
            full_name=full_name,
            twenty_four_hour_time=False,
            groups=[],  # This is Zulip-specific
            enable_online_push_notifications=False,
            alert_words="[]",
            bot_owner=None,  # This is Zulip-specific
            short_name=user['name'],
            enable_offline_push_notifications=True,
            left_side_userlist=False,
            enable_stream_desktop_notifications=False,
            enable_digest_emails=True,
            last_pointer_updater="",
            email=email,
            date_joined=timestamp,
            last_reminder=timestamp,
            is_superuser=False,
            tutorial_status="T",
            default_language="en",
            enable_sounds=True,
            pm_content_in_desktop_notifications=True,
            is_active=not user['deleted'],
            onboarding_steps="[]",
            emojiset="google",
            emoji_alt_code=False,
            realm=realm_id,
            quota=1073741824,
            # invites_used=0,  # TODO
            id=user_id_count)

        # TODO map the avatar
        # zerver auto-infer the url from Gravatar instead of from a specified
        # url; zerver.lib.avatar needs to be patched
        # profile['image_32'], Slack has 24, 32, 48, 72, 192, 512 size range

        zerver_userprofile.append(userprofile)
        added_users[slack_user_id] = user_id_count
        user_id_count += 1
        print(u"{} -> {}\nCreated\n".format(user['name'], userprofile['email']))
    print('######### IMPORTING USERS FINISHED #########\n')
    return zerver_userprofile, added_users

def channels2zerver_stream(slack_dir, realm_id, added_users, zerver_userprofile):
    # type: (Dict[str, Dict[str, Any]]) -> None
    print('######### IMPORTING CHANNELS STARTED #########\n')
    channels = json.load(open(slack_dir + '/channels.json'))
    added_channels = {}
    zerver_stream = []
    stream_id_count = 1
    zerver_subscription = []
    zerver_recipient = []
    subscription_id_count = 1
    zerver_defaultstream = []

    for channel in channels:
        # slack_channel_id = channel['id']

        # map Slack's topic and purpose content into Zulip's stream description.
        # WARN This mapping is lossy since the topic.creator, topic.last_set,
        # purpose.creator, purpose.last_set fields are not preserved.
        description = "topic: {}\npurpose: {}".format(channel["topic"]["value"],
                                                      channel["purpose"]["value"])

        # construct the stream object and append it to zerver_stream
        stream = dict(
            realm=realm_id,
            name=channel["name"],
            deactivated=channel["is_archived"],
            description=description,
            invite_only=not channel["is_general"],
            date_created=float(channel["created"]),
            id=stream_id_count)

        if channel["name"] == "general":
            defaultstream = dict(
                stream=stream_id_count,
                realm=realm_id,
                id=1)
            zerver_defaultstream.append(defaultstream)

        zerver_stream.append(stream)
        added_channels[stream['name']] = stream_id_count

        # construct the recipient object and append it to zerver_recipient
        # type 1: private
        # type 2: stream
        # type 3: huddle
        recipient = dict(
            type_id=stream_id_count,
            id=stream_id_count,
            type=2)
        zerver_recipient.append(recipient)
        # TOODO add recipients for private message and huddles

        # construct the subscription object and append it to zerver_subscription
        for member in channel['members']:
            sub = dict(
                recipient=stream_id_count,
                notifications=False,
                color="#c2c2c2",
                desktop_notifications=True,
                pin_to_top=False,
                in_home_view=True,
                active=True,
                user_profile=added_users[member],
                id=subscription_id_count)
            # The recipient is a stream for stream-readable message.
            # proof :  https://github.com/zulip/zulip/blob/master/zerver/views/messages.py#L240 &
            # https://github.com/zulip/zulip/blob/master/zerver/views/messages.py#L324
            zerver_subscription.append(sub)
            subscription_id_count += 1
            # TOODO add zerver_subscription which correspond to
            # huddles type recipient
            # For huddles:
            # sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member]

            # TOODO do private message subscriptions between each users have to
            # be generated from scratch?

        stream_id_count += 1
        print(u"{} -> created\n".format(channel['name']))

        # TODO map Slack's pins to Zulip's stars
        # There is the security model that Slack's pins are known to the team owner
        # as evident from where it is stored at (channels)
        # "pins": [
        #         {
        #             "id": "1444755381.000003",
        #             "type": "C",
        #             "user": "U061A5N1G",
        #             "owner": "U061A5N1G",
        #             "created": "1444755463"
        #         }
        #         ],
    recipient_id_count = stream_id_count + 1
    subscription_id_count += 1

    for user in zerver_userprofile:
        zulip_user_id = user['id']

        # this maps the recipients and subscriptions
        # related to private messages

        recipient = dict(
            type_id=zulip_user_id,
            id=recipient_id_count,
            type=1)
        zerver_recipient.append(recipient)

        sub = dict(
            recipient=recipient_id_count,
            notifications=False,
            color="#c2c2c2",
            desktop_notifications=True,
            pin_to_top=False,
            in_home_view=True,
            active=True,
            user_profile=zulip_user_id,
            id=subscription_id_count)

        zerver_subscription.append(sub)
        subscription_id_count += 1
        recipient_id_count += 1

    print('######### IMPORTING STREAMS FINISHED #########\n')
    return zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, zerver_recipient

def channelmessage2zerver_message_one_stream(constants, channel, added_users,
                                             zerver_userprofile,
                                             added_channels, ids,
                                             zerver_subscription):
    slack_dir, REALM_ID, upload_dir = constants
    message_id, usermessage_id, attachment_id = ids
    json_names = os.listdir(slack_dir + '/' + channel)
    users = json.load(open(slack_dir + '/users.json'))
    zerver_message = []
    zerver_usermessage = []
    zerver_attachment = []

    # Sanitize the message text
    def sanitize_text(text):
        tokens = text.split(' ')
        text = ' '.join([sanitize_token(t) for t in tokens])
        return text

    def sanitize_token(token):
        if (re.compile(r"<@.*|.*>").match(token)):
            token = token.replace('<@', ' ')
            token = token.replace('>', ' ')
            token = token.replace('|', ' ')
            morphemes = token.split(' ')
            length = len(morphemes)
            if length > 1:
                if length > 2:
                    short_name = morphemes[2]
                else:
                    short_name = ''
                token = morphemes[1]
            for user in users:
                if (user['id'] == token and user['name'] == short_name and length == 4) or \
                   (user['id'] == token and length == 3):
                    token = user.get('real_name', user['name'])
                    token = "@**" + token + "** "
        return token

    # check if a user has been mentioned in a message
    # for zerver_usermessage
    def check_user_mention(text):
        # Zulip's at mention
        mentions = re.findall(r'(@(?:\*\*([^\*]+)\*\*|(\w+)))', text)
        mentioned_users_id = []
        for mention in mentions:
            for userprofile in zerver_userprofile:
                if mention[1] == userprofile['full_name']:
                        mentioned_users_id.append(userprofile['id'])
        return mentioned_users_id

    # check if the text contain a URL
    def check_has_link(msg):
        if 'has_link' in msg:
            return msg['has_link']
        else:
            # TODO map msg['attachments']['from_url']
            text = msg['text']
            return ('http://' in text or 'https://' in text)

    def parse_url(url):
        return url.replace("\/\/", "//").replace("\/", "/")

    def save_attachment(url, _id, name):
        url = parse_url(url)
        response = requests.get(url, stream=True)
        os.makedirs(upload_dir + '/' + str(id), exist_ok=True)
        with open(upload_dir + '/' + str(id) + '/' + name, 'wb') as output_file:
            shutil.copyfileobj(response.raw, output_file)

    for json_name in json_names:
        msgs = json.load(open(slack_dir + '/%s/%s' % (channel, json_name)))
        for msg in msgs:
            text = msg['text']
            has_attachment = False

            if 'subtype' in msg.keys():
                st = msg['subtype']
                if st in ["channel_join", "channel_leave", "channel_name"]:
                    # Ignore noisy messages
                    continue
                elif st == "file_share":
                    has_attachment = True
                    _file = msg['file']
                    slack_user_id = _file['user']
                    zulip_user_id = added_users[slack_user_id]
                    save_attachment(_file['url_private'], attachment_id, _file['name'])
                    path_id = "%d\/%d\/%s" % (REALM_ID, attachment_id, _file['name'])
                    # construct attachments object and append it to zerver_attachment
                    attachments = dict(
                        id=attachment_id,
                        is_realm_public=True,  # TOODOO map for private messages and huddles, where is_realm_public = False
                        file_name=_file['name'],
                        create_time=_file['created'],
                        size=_file['size'],
                        path_id=path_id,
                        realm=REALM_ID,
                        owner=zulip_user_id,
                        messages=[message_id])
                    attachment_id += 1
                    zerver_attachment.append(attachments)

            try:
                user = msg.get('user', msg['file']['user'])
            except KeyError:
                # black magic, explain this later TOODOO
                user = msg['user']
            # construct message
            zulip_message = dict(
                sending_client=1,
                rendered_content_version=1,  # This is Zulip-specific
                has_image=msg.get('has_image', False),
                subject=channel,  # This is Zulip-specific
                pub_date=msg['ts'],
                id=message_id,
                has_attachment=has_attachment,  # attachment will be posted in the subsequent message;
                                                # this is how Slack does it, i.e. less like email
                edit_history=None,
                sender=added_users[user],  # map slack id to zulip id
                content=sanitize_text(text),
                rendered_content=text,  # slack doesn't cache this
                recipient=added_channels[channel],
                last_edit_time=None,
                has_link=check_has_link(msg))
            zerver_message.append(zulip_message)

            # construct usermessages
            mentioned_users_id = check_user_mention(zulip_message['content'])
            for subscription in zerver_subscription:
                if subscription['recipient'] == zulip_message['recipient']:
                    flags_mask = 1
                    if subscription['user_profile'] in mentioned_users_id:
                        flags_mask = 9

                    usermessage = dict(
                        user_profile=subscription['user_profile'],
                        id=usermessage_id,
                        flags_mask=flags_mask,  # defaulting to 'read' or 'mentioned' and 'read'
                        message=zulip_message['id'])
                    usermessage_id += 1
                    zerver_usermessage.append(usermessage)

            message_id += 1
    return zerver_message, zerver_usermessage, zerver_attachment

def main(slack_zip_file: str) -> None:
    slack_dir = slack_zip_file.replace('.zip', '')
    subprocess.check_call(['unzip', slack_zip_file])
    # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref:
    #     zip_ref.extractall(slack_dir)

    from datetime import datetime
    # TODO fetch realm config from zulip config
    DOMAIN_NAME = "zulipchat.com"
    # Hardcode this to 1, will implement later for zulipchat.com's case where it
    # has multiple realms
    REALM_ID = 1
    REALM_NAME = "FleshEatingBatswithFangs"
    NOW = float(datetime.utcnow().timestamp())

    script_path = os.path.dirname(os.path.abspath(__file__)) + '/'
    zerver_realm_skeleton = json.load(open(script_path + 'zerver_realm_skeleton.json'))
    zerver_realm_skeleton[0]['id'] = REALM_ID
    zerver_realm_skeleton[0]['string_id'] = 'zulip'  # subdomain / short_name of realm
    zerver_realm_skeleton[0]['name'] = REALM_NAME
    zerver_realm_skeleton[0]['date_created'] = NOW

    # Make sure the directory output is clean
    output_dir = 'zulip_data'
    rm_tree(output_dir)
    os.makedirs(output_dir, exist_ok=True)

    realm = dict(zerver_client=[{"name": "populate_db", "id": 1},
                                {"name": "website", "id": 2},
                                {"name": "API", "id": 3}],
                 zerver_userpresence=[],  # shows last logged in data, which is not available in slack
                 zerver_userprofile_mirrordummy=[],
                 zerver_realmdomain=[{"realm": REALM_ID,
                                      "allow_subdomains": False,
                                      "domain": DOMAIN_NAME,
                                      "id": REALM_ID}],
                 zerver_useractivity=[],
                 zerver_realm=zerver_realm_skeleton,
                 zerver_huddle=[],
                 zerver_userprofile_crossrealm=[],
                 zerver_useractivityinterval=[],
                 zerver_realmfilter=[],
                 zerver_realmemoji=[])

    zerver_userprofile, added_users = users2zerver_userprofile(slack_dir,
                                                               REALM_ID,
                                                               int(NOW),
                                                               DOMAIN_NAME)
    realm['zerver_userprofile'] = zerver_userprofile

    zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, zerver_recipient = channels2zerver_stream(slack_dir,
                                                                                                                        REALM_ID,
                                                                                                                        added_users,
                                                                                                                        zerver_userprofile)
    # See https://zulipchat.com/help/set-default-streams-for-new-users
    # for documentation on zerver_defaultstream
    realm['zerver_defaultstream'] = zerver_defaultstream
    realm['zerver_stream'] = zerver_stream
    realm['zerver_subscription'] = zerver_subscription
    realm['zerver_recipient'] = zerver_recipient
    # IO realm.json
    realm_file = output_dir + '/realm.json'
    json.dump(realm, open(realm_file, 'w'))

    # now for message.json
    message_json = {}
    zerver_message = []
    zerver_usermessage = []
    zerver_attachment = []

    upload_dir = output_dir + '/uploads/' + str(REALM_ID)
    constants = [slack_dir, REALM_ID, upload_dir]
    for channel in added_channels.keys():
        message_id = len(zerver_message) + 1  # For the id of the messages
        usermessage_id = len(zerver_usermessage) + 1
        attachment_id = len(zerver_attachment) + 1
        ids = [message_id, usermessage_id, attachment_id]
        zm, zum, za = channelmessage2zerver_message_one_stream(constants, channel,
                                                               added_users, zerver_userprofile,
                                                               added_channels, ids,
                                                               zerver_subscription)
        zerver_message += zm
        zerver_usermessage += zum
        zerver_attachment += za
        # TOODOO add zerver_usermessage corresponding to the
        # private messages and huddles type recipients

    message_json['zerver_message'] = zerver_message
    message_json['zerver_usermessage'] = zerver_usermessage
    # IO message.json
    message_file = output_dir + '/message.json'
    json.dump(message_json, open(message_file, 'w'))

    # IO avatar records
    avatar_records_file = output_dir + '/avatars/records.json'
    os.makedirs(output_dir + '/avatars', exist_ok=True)
    json.dump([], open(avatar_records_file, 'w'))

    # IO uploads TODO
    uploads_records_file = output_dir + '/uploads/records.json'
    os.makedirs(output_dir + '/uploads', exist_ok=True)
    json.dump([], open(uploads_records_file, 'w'))

    # IO attachments
    attachment_file = output_dir + '/attachment.json'
    # attachment = {"zerver_attachment": zerver_attachment}
    attachment = {"zerver_attachment": []}
    json.dump(attachment, open(attachment_file, 'w'))

    print('ls', os.listdir())
    print('pwd', os.getcwd())

    # remove slack dir
    rm_tree(slack_dir)

    # compress the folder
    subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir])
    # subprocess.check_call(['zip', '-jpr', output_dir + '.zip', realm_file, message_file])

    # remove zulip dir
    rm_tree(output_dir)

    sys.exit(0)

if __name__ == '__main__':
    # from django.conf import settings
    # settings_module = "settings.py"
    # os.environ['DJANGO_SETTINGS_MODULE'] = settings_module
    description = ("script to convert Slack export data into Zulip export data")
    parser = argparse.ArgumentParser(description=description)
    slack_zip_file = sys.argv[1]
    main(slack_zip_file)