From 0bb50fdc4ee9d4a4486664752dd4d179ce9be4c3 Mon Sep 17 00:00:00 2001 From: Rhea Parekh Date: Wed, 13 Dec 2017 12:19:47 +0530 Subject: [PATCH] slack importer: Shift the slack data conversion files to zulip server repo. --- .travis.yml | 2 - tools/test-slack-importer | 19 - .../integrations/slack/slackdata2zulipdata.py | 377 ------------------ .../slack/zerver_realm_skeleton.json | 36 -- 4 files changed, 434 deletions(-) delete mode 100755 tools/test-slack-importer delete mode 100755 zulip/integrations/slack/slackdata2zulipdata.py delete mode 100644 zulip/integrations/slack/zerver_realm_skeleton.json diff --git a/.travis.yml b/.travis.yml index 0fe342a..9c4997f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,6 @@ matrix: env: TEST_SUITE=test-py2-legacy-support - python: "3.6" env: TEST_SUITE=test-static-analysis - - python: "3.4" - env: TEST_SUITE=test-slack-importer addons: postgresql: "9.3" install: diff --git a/tools/test-slack-importer b/tools/test-slack-importer deleted file mode 100755 index 78be948..0000000 --- a/tools/test-slack-importer +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -ev - -wget https://github.com/houstondatavis/slack-export/archive/master.zip - -mv master.zip slack-export-master.zip -zulip/integrations/slack/slackdata2zulipdata.py $PWD/slack-export-master.zip - -mv zulip_data.tar.gz $HOME -cd -git clone --depth=50 --branch=test-slack-importer https://github.com/rheaparekh/zulip.git -cd zulip -mv ../zulip_data.tar.gz . -tar xf zulip_data.tar.gz - -./tools/travis/setup-backend -source tools/travis/activate-venv -./manage.py import --destroy-rebuild-database $PWD/zulip_data diff --git a/zulip/integrations/slack/slackdata2zulipdata.py b/zulip/integrations/slack/slackdata2zulipdata.py deleted file mode 100755 index 6c3059d..0000000 --- a/zulip/integrations/slack/slackdata2zulipdata.py +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env python -import os -import json -import hashlib -import sys -import argparse -import shutil -import subprocess -import re - -from typing import Any, Dict, List, Tuple -# stubs -ZerverFieldsT = Dict[str, Any] -AddedUsersT = Dict[str, int] -AddedChannelsT = Dict[str, int] - -# Transported from https://github.com/zulip/zulip/blob/master/zerver/lib/export.py -def rm_tree(path: str) -> None: - if os.path.exists(path): - shutil.rmtree(path) - -def users2zerver_userprofile(slack_dir: str, realm_id: int, timestamp: Any, - domain_name: str) -> Tuple[List[ZerverFieldsT], AddedUsersT]: - """ - Returns: - 1. zerver_userprofile, which is a list of user profile - 2. added_users, which is a dictionary to map from slack user id to zulip - user id - """ - print('######### IMPORTING USERS STARTED #########\n') - users = json.load(open(slack_dir + '/users.json')) - zerver_userprofile = [] - added_users = {} - user_id_count = 1 - for user in users: - slack_user_id = user['id'] - profile = user['profile'] - DESKTOP_NOTIFICATION = True - - # email - if 'email' not in profile: - email = (hashlib.sha256(user['real_name'].encode()).hexdigest() + - "@%s" % (domain_name)) - else: - email = profile['email'] - - # avatar - # ref: https://chat.zulip.org/help/change-your-avatar - avatar_source = 'U' - if 'gravatar.com' in profile['image_32']: - # use the avatar from gravatar - avatar_source = 'G' - - # timezone - _default_timezone = "America/New_York" - timezone = user.get("tz", _default_timezone) - if timezone is None or '/' not in timezone: - timezone = _default_timezone - - if user['deleted'] is False: - if user['real_name'] == '': - full_name = user['name'] - else: - full_name = user['real_name'] - - # userprofile's quota is hardcoded as per - # https://github.com/zulip/zulip/blob/e1498988d9094961e6f9988fb308b3e7310a8e74/zerver/migrations/0059_userprofile_quota.py#L18 - userprofile = dict( - enable_desktop_notifications=DESKTOP_NOTIFICATION, - is_staff=user.get('is_admin', False), - avatar_source=avatar_source, - is_bot=user.get('is_bot', False), - avatar_version=1, - autoscroll_forever=False, - default_desktop_notifications=True, - timezone=timezone, - default_sending_stream=None, - enable_offline_email_notifications=True, - user_permissions=[], # This is Zulip-specific - is_mirror_dummy=False, - pointer=-1, - default_events_register_stream=None, - is_realm_admin=user.get('is_owner', False), - # invites_granted=0, # TODO - enter_sends=True, - bot_type=1 if user.get('is_bot', False) else None, - enable_stream_sounds=False, - is_api_super_user=False, - rate_limits="", - last_login=timestamp, - tos_version=None, - default_all_public_streams=False, - full_name=full_name, - twenty_four_hour_time=False, - groups=[], # This is Zulip-specific - enable_online_push_notifications=False, - alert_words="[]", - bot_owner=None, # This is Zulip-specific - short_name=user['name'], - enable_offline_push_notifications=True, - left_side_userlist=False, - enable_stream_desktop_notifications=False, - enable_digest_emails=True, - last_pointer_updater="", - email=email, - date_joined=timestamp, - last_reminder=timestamp, - is_superuser=False, - tutorial_status="T", - default_language="en", - enable_sounds=True, - pm_content_in_desktop_notifications=True, - is_active=not user['deleted'], - onboarding_steps="[]", - emojiset="google", - emoji_alt_code=False, - realm=realm_id, - quota=1073741824, - # invites_used=0, # TODO - id=user_id_count) - - # TODO map the avatar - # zerver auto-infer the url from Gravatar instead of from a specified - # url; zerver.lib.avatar needs to be patched - # profile['image_32'], Slack has 24, 32, 48, 72, 192, 512 size range - - zerver_userprofile.append(userprofile) - added_users[slack_user_id] = user_id_count - user_id_count += 1 - print(u"{} -> {}\nCreated\n".format(user['name'], userprofile['email'])) - print('######### IMPORTING USERS FINISHED #########\n') - return zerver_userprofile, added_users - -def channels2zerver_stream(slack_dir: str, realm_id: int, added_users: AddedUsersT, - zerver_userprofile: List[ZerverFieldsT]) -> Tuple[List[ZerverFieldsT], - List[ZerverFieldsT], - AddedChannelsT, - List[ZerverFieldsT], - List[ZerverFieldsT]]: - print('######### IMPORTING CHANNELS STARTED #########\n') - channels = json.load(open(slack_dir + '/channels.json')) - added_channels = {} - zerver_stream = [] - stream_id_count = 1 - zerver_subscription = [] - zerver_recipient = [] - subscription_id_count = 1 - zerver_defaultstream = [] - - for channel in channels: - # slack_channel_id = channel['id'] - - # map Slack's topic and purpose content into Zulip's stream description. - # WARN This mapping is lossy since the topic.creator, topic.last_set, - # purpose.creator, purpose.last_set fields are not preserved. - description = "topic: {}\npurpose: {}".format(channel["topic"]["value"], - channel["purpose"]["value"]) - - # construct the stream object and append it to zerver_stream - stream = dict( - realm=realm_id, - name=channel["name"], - deactivated=channel["is_archived"], - description=description, - invite_only=not channel["is_general"], - date_created=float(channel["created"]), - id=stream_id_count) - - if channel["name"] == "general": - defaultstream = dict( - stream=stream_id_count, - realm=realm_id, - id=1) - zerver_defaultstream.append(defaultstream) - - zerver_stream.append(stream) - added_channels[stream['name']] = stream_id_count - - # construct the recipient object and append it to zerver_recipient - # type 1: private - # type 2: stream - # type 3: huddle - recipient = dict( - type_id=stream_id_count, - id=stream_id_count, - type=2) - zerver_recipient.append(recipient) - # TOODO add recipients for private message and huddles - - # construct the subscription object and append it to zerver_subscription - for member in channel['members']: - sub = dict( - recipient=stream_id_count, - notifications=False, - color="#c2c2c2", - desktop_notifications=True, - pin_to_top=False, - in_home_view=True, - active=True, - user_profile=added_users[member], - id=subscription_id_count) - # The recipient is a stream for stream-readable message. - # proof : https://github.com/zulip/zulip/blob/master/zerver/views/messages.py#L240 & - # https://github.com/zulip/zulip/blob/master/zerver/views/messages.py#L324 - zerver_subscription.append(sub) - subscription_id_count += 1 - # TOODO add zerver_subscription which correspond to - # huddles type recipient - # For huddles: - # sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member] - - # TOODO do private message subscriptions between each users have to - # be generated from scratch? - - stream_id_count += 1 - print(u"{} -> created\n".format(channel['name'])) - - # TODO map Slack's pins to Zulip's stars - # There is the security model that Slack's pins are known to the team owner - # as evident from where it is stored at (channels) - # "pins": [ - # { - # "id": "1444755381.000003", - # "type": "C", - # "user": "U061A5N1G", - # "owner": "U061A5N1G", - # "created": "1444755463" - # } - # ], - recipient_id_count = stream_id_count + 1 - subscription_id_count += 1 - - for user in zerver_userprofile: - zulip_user_id = user['id'] - - # this maps the recipients and subscriptions - # related to private messages - - recipient = dict( - type_id=zulip_user_id, - id=recipient_id_count, - type=1) - zerver_recipient.append(recipient) - - sub = dict( - recipient=recipient_id_count, - notifications=False, - color="#c2c2c2", - desktop_notifications=True, - pin_to_top=False, - in_home_view=True, - active=True, - user_profile=zulip_user_id, - id=subscription_id_count) - - zerver_subscription.append(sub) - subscription_id_count += 1 - recipient_id_count += 1 - - print('######### IMPORTING STREAMS FINISHED #########\n') - return zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, zerver_recipient - -def main(slack_zip_file: str) -> None: - slack_dir = slack_zip_file.replace('.zip', '') - subprocess.check_call(['unzip', slack_zip_file]) - # with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref: - # zip_ref.extractall(slack_dir) - - from datetime import datetime - # TODO fetch realm config from zulip config - DOMAIN_NAME = "zulipchat.com" - - # Hardcode this to 1, will implement later for zulipchat.com's case where it - # has multiple realms - REALM_ID = 1 - REALM_NAME = "FleshEatingBatswithFangs" - NOW = float(datetime.utcnow().timestamp()) - - script_path = os.path.dirname(os.path.abspath(__file__)) + '/' - zerver_realm_skeleton = json.load(open(script_path + 'zerver_realm_skeleton.json')) - zerver_realm_skeleton[0]['id'] = REALM_ID - zerver_realm_skeleton[0]['string_id'] = 'zulip' # subdomain / short_name of realm - zerver_realm_skeleton[0]['name'] = REALM_NAME - zerver_realm_skeleton[0]['date_created'] = NOW - - # Make sure the directory output is clean - output_dir = 'zulip_data' - rm_tree(output_dir) - os.makedirs(output_dir, exist_ok=True) - - realm = dict(zerver_client=[{"name": "populate_db", "id": 1}, - {"name": "website", "id": 2}, - {"name": "API", "id": 3}], - zerver_userpresence=[], # shows last logged in data, which is not available in slack - zerver_userprofile_mirrordummy=[], - zerver_realmdomain=[{"realm": REALM_ID, - "allow_subdomains": False, - "domain": DOMAIN_NAME, - "id": REALM_ID}], - zerver_useractivity=[], - zerver_realm=zerver_realm_skeleton, - zerver_huddle=[], - zerver_userprofile_crossrealm=[], - zerver_useractivityinterval=[], - zerver_realmfilter=[], - zerver_realmemoji=[]) - - zerver_userprofile, added_users = users2zerver_userprofile(slack_dir, - REALM_ID, - int(NOW), - DOMAIN_NAME) - realm['zerver_userprofile'] = zerver_userprofile - - zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, zerver_recipient = channels2zerver_stream(slack_dir, - REALM_ID, - added_users, - zerver_userprofile) - # See https://zulipchat.com/help/set-default-streams-for-new-users - # for documentation on zerver_defaultstream - realm['zerver_defaultstream'] = zerver_defaultstream - realm['zerver_stream'] = zerver_stream - realm['zerver_subscription'] = zerver_subscription - realm['zerver_recipient'] = zerver_recipient - # IO realm.json - realm_file = output_dir + '/realm.json' - json.dump(realm, open(realm_file, 'w')) - - # now for message.json - message_json = {} - zerver_message = [] # type: List[ZerverFieldsT] - zerver_usermessage = [] # type: List[ZerverFieldsT] - zerver_attachment = [] # type: List[ZerverFieldsT] - - message_json['zerver_message'] = zerver_message - message_json['zerver_usermessage'] = zerver_usermessage - # IO message.json - message_file = output_dir + '/message.json' - json.dump(message_json, open(message_file, 'w')) - - # IO avatar records - avatar_records_file = output_dir + '/avatars/records.json' - os.makedirs(output_dir + '/avatars', exist_ok=True) - json.dump([], open(avatar_records_file, 'w')) - - # IO uploads TODO - uploads_records_file = output_dir + '/uploads/records.json' - os.makedirs(output_dir + '/uploads', exist_ok=True) - json.dump([], open(uploads_records_file, 'w')) - - # IO attachments - attachment_file = output_dir + '/attachment.json' - attachment = {"zerver_attachment": zerver_attachment} - json.dump(attachment, open(attachment_file, 'w')) - - print('ls', os.listdir()) - print('pwd', os.getcwd()) - - # remove slack dir - rm_tree(slack_dir) - - # compress the folder - subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir]) - # subprocess.check_call(['zip', '-jpr', output_dir + '.zip', realm_file, message_file]) - - # remove zulip dir - rm_tree(output_dir) - - sys.exit(0) - -if __name__ == '__main__': - # from django.conf import settings - # settings_module = "settings.py" - # os.environ['DJANGO_SETTINGS_MODULE'] = settings_module - description = ("script to convert Slack export data into Zulip export data") - parser = argparse.ArgumentParser(description=description) - slack_zip_file = sys.argv[1] - main(slack_zip_file) diff --git a/zulip/integrations/slack/zerver_realm_skeleton.json b/zulip/integrations/slack/zerver_realm_skeleton.json deleted file mode 100644 index 491e15b..0000000 --- a/zulip/integrations/slack/zerver_realm_skeleton.json +++ /dev/null @@ -1,36 +0,0 @@ -[{ - "message_retention_days": null, - "inline_image_preview": true, - "name_changes_disabled": false, - "string_id": "zulip", - "icon_version": 1, - "waiting_period_threshold": 0, - "email_changes_disabled": false, - "deactivated": false, - "notifications_stream": null, - "restricted_to_domain": true, - "show_digest_email": true, - "allow_message_editing": true, - "description": "The Zulip development environment default organization. It's great for testing!", - "default_language": "en", - "icon_source": "G", - "invite_required": false, - "invite_by_admins_only": false, - "create_stream_by_admins_only": false, - "mandatory_topics": false, - "inline_url_embed_preview": true, - "message_content_edit_limit_seconds": 600, - "authentication_methods": [ - ["Google", true], - ["Email", true], - ["GitHub", true], - ["LDAP", true], - ["Dev", true], - ["RemoteUser", true] - ], - "name": "", - "org_type": 1, - "add_emoji_by_admins_only": false, - "date_created": 0.0, - "id": 1 -}]