slack importer: Shift the slack data conversion files to zulip server repo.
This commit is contained in:
Rhea Parekh
showell
parent
1599839956
commit
0bb50fdc4e
|
|
@ -11,8 +11,6 @@ matrix:
|
||||||
env: TEST_SUITE=test-py2-legacy-support
|
env: TEST_SUITE=test-py2-legacy-support
|
||||||
- python: "3.6"
|
- python: "3.6"
|
||||||
env: TEST_SUITE=test-static-analysis
|
env: TEST_SUITE=test-static-analysis
|
||||||
- python: "3.4"
|
|
||||||
env: TEST_SUITE=test-slack-importer
|
|
||||||
addons:
|
addons:
|
||||||
postgresql: "9.3"
|
postgresql: "9.3"
|
||||||
install:
|
install:
|
||||||
|
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -ev
|
|
||||||
|
|
||||||
wget https://github.com/houstondatavis/slack-export/archive/master.zip
|
|
||||||
|
|
||||||
mv master.zip slack-export-master.zip
|
|
||||||
zulip/integrations/slack/slackdata2zulipdata.py $PWD/slack-export-master.zip
|
|
||||||
|
|
||||||
mv zulip_data.tar.gz $HOME
|
|
||||||
cd
|
|
||||||
git clone --depth=50 --branch=test-slack-importer https://github.com/rheaparekh/zulip.git
|
|
||||||
cd zulip
|
|
||||||
mv ../zulip_data.tar.gz .
|
|
||||||
tar xf zulip_data.tar.gz
|
|
||||||
|
|
||||||
./tools/travis/setup-backend
|
|
||||||
source tools/travis/activate-venv
|
|
||||||
./manage.py import --destroy-rebuild-database $PWD/zulip_data
|
|
||||||
|
|
@ -1,377 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import hashlib
|
|
||||||
import sys
|
|
||||||
import argparse
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import re
|
|
||||||
|
|
||||||
from typing import Any, Dict, List, Tuple
|
|
||||||
# stubs
|
|
||||||
ZerverFieldsT = Dict[str, Any]
|
|
||||||
AddedUsersT = Dict[str, int]
|
|
||||||
AddedChannelsT = Dict[str, int]
|
|
||||||
|
|
||||||
# Transported from https://github.com/zulip/zulip/blob/master/zerver/lib/export.py
|
|
||||||
def rm_tree(path: str) -> None:
|
|
||||||
if os.path.exists(path):
|
|
||||||
shutil.rmtree(path)
|
|
||||||
|
|
||||||
def users2zerver_userprofile(slack_dir: str, realm_id: int, timestamp: Any,
|
|
||||||
domain_name: str) -> Tuple[List[ZerverFieldsT], AddedUsersT]:
|
|
||||||
"""
|
|
||||||
Returns:
|
|
||||||
1. zerver_userprofile, which is a list of user profile
|
|
||||||
2. added_users, which is a dictionary to map from slack user id to zulip
|
|
||||||
user id
|
|
||||||
"""
|
|
||||||
print('######### IMPORTING USERS STARTED #########\n')
|
|
||||||
users = json.load(open(slack_dir + '/users.json'))
|
|
||||||
zerver_userprofile = []
|
|
||||||
added_users = {}
|
|
||||||
user_id_count = 1
|
|
||||||
for user in users:
|
|
||||||
slack_user_id = user['id']
|
|
||||||
profile = user['profile']
|
|
||||||
DESKTOP_NOTIFICATION = True
|
|
||||||
|
|
||||||
# email
|
|
||||||
if 'email' not in profile:
|
|
||||||
email = (hashlib.sha256(user['real_name'].encode()).hexdigest() +
|
|
||||||
"@%s" % (domain_name))
|
|
||||||
else:
|
|
||||||
email = profile['email']
|
|
||||||
|
|
||||||
# avatar
|
|
||||||
# ref: https://chat.zulip.org/help/change-your-avatar
|
|
||||||
avatar_source = 'U'
|
|
||||||
if 'gravatar.com' in profile['image_32']:
|
|
||||||
# use the avatar from gravatar
|
|
||||||
avatar_source = 'G'
|
|
||||||
|
|
||||||
# timezone
|
|
||||||
_default_timezone = "America/New_York"
|
|
||||||
timezone = user.get("tz", _default_timezone)
|
|
||||||
if timezone is None or '/' not in timezone:
|
|
||||||
timezone = _default_timezone
|
|
||||||
|
|
||||||
if user['deleted'] is False:
|
|
||||||
if user['real_name'] == '':
|
|
||||||
full_name = user['name']
|
|
||||||
else:
|
|
||||||
full_name = user['real_name']
|
|
||||||
|
|
||||||
# userprofile's quota is hardcoded as per
|
|
||||||
# https://github.com/zulip/zulip/blob/e1498988d9094961e6f9988fb308b3e7310a8e74/zerver/migrations/0059_userprofile_quota.py#L18
|
|
||||||
userprofile = dict(
|
|
||||||
enable_desktop_notifications=DESKTOP_NOTIFICATION,
|
|
||||||
is_staff=user.get('is_admin', False),
|
|
||||||
avatar_source=avatar_source,
|
|
||||||
is_bot=user.get('is_bot', False),
|
|
||||||
avatar_version=1,
|
|
||||||
autoscroll_forever=False,
|
|
||||||
default_desktop_notifications=True,
|
|
||||||
timezone=timezone,
|
|
||||||
default_sending_stream=None,
|
|
||||||
enable_offline_email_notifications=True,
|
|
||||||
user_permissions=[], # This is Zulip-specific
|
|
||||||
is_mirror_dummy=False,
|
|
||||||
pointer=-1,
|
|
||||||
default_events_register_stream=None,
|
|
||||||
is_realm_admin=user.get('is_owner', False),
|
|
||||||
# invites_granted=0, # TODO
|
|
||||||
enter_sends=True,
|
|
||||||
bot_type=1 if user.get('is_bot', False) else None,
|
|
||||||
enable_stream_sounds=False,
|
|
||||||
is_api_super_user=False,
|
|
||||||
rate_limits="",
|
|
||||||
last_login=timestamp,
|
|
||||||
tos_version=None,
|
|
||||||
default_all_public_streams=False,
|
|
||||||
full_name=full_name,
|
|
||||||
twenty_four_hour_time=False,
|
|
||||||
groups=[], # This is Zulip-specific
|
|
||||||
enable_online_push_notifications=False,
|
|
||||||
alert_words="[]",
|
|
||||||
bot_owner=None, # This is Zulip-specific
|
|
||||||
short_name=user['name'],
|
|
||||||
enable_offline_push_notifications=True,
|
|
||||||
left_side_userlist=False,
|
|
||||||
enable_stream_desktop_notifications=False,
|
|
||||||
enable_digest_emails=True,
|
|
||||||
last_pointer_updater="",
|
|
||||||
email=email,
|
|
||||||
date_joined=timestamp,
|
|
||||||
last_reminder=timestamp,
|
|
||||||
is_superuser=False,
|
|
||||||
tutorial_status="T",
|
|
||||||
default_language="en",
|
|
||||||
enable_sounds=True,
|
|
||||||
pm_content_in_desktop_notifications=True,
|
|
||||||
is_active=not user['deleted'],
|
|
||||||
onboarding_steps="[]",
|
|
||||||
emojiset="google",
|
|
||||||
emoji_alt_code=False,
|
|
||||||
realm=realm_id,
|
|
||||||
quota=1073741824,
|
|
||||||
# invites_used=0, # TODO
|
|
||||||
id=user_id_count)
|
|
||||||
|
|
||||||
# TODO map the avatar
|
|
||||||
# zerver auto-infer the url from Gravatar instead of from a specified
|
|
||||||
# url; zerver.lib.avatar needs to be patched
|
|
||||||
# profile['image_32'], Slack has 24, 32, 48, 72, 192, 512 size range
|
|
||||||
|
|
||||||
zerver_userprofile.append(userprofile)
|
|
||||||
added_users[slack_user_id] = user_id_count
|
|
||||||
user_id_count += 1
|
|
||||||
print(u"{} -> {}\nCreated\n".format(user['name'], userprofile['email']))
|
|
||||||
print('######### IMPORTING USERS FINISHED #########\n')
|
|
||||||
return zerver_userprofile, added_users
|
|
||||||
|
|
||||||
def channels2zerver_stream(slack_dir: str, realm_id: int, added_users: AddedUsersT,
|
|
||||||
zerver_userprofile: List[ZerverFieldsT]) -> Tuple[List[ZerverFieldsT],
|
|
||||||
List[ZerverFieldsT],
|
|
||||||
AddedChannelsT,
|
|
||||||
List[ZerverFieldsT],
|
|
||||||
List[ZerverFieldsT]]:
|
|
||||||
print('######### IMPORTING CHANNELS STARTED #########\n')
|
|
||||||
channels = json.load(open(slack_dir + '/channels.json'))
|
|
||||||
added_channels = {}
|
|
||||||
zerver_stream = []
|
|
||||||
stream_id_count = 1
|
|
||||||
zerver_subscription = []
|
|
||||||
zerver_recipient = []
|
|
||||||
subscription_id_count = 1
|
|
||||||
zerver_defaultstream = []
|
|
||||||
|
|
||||||
for channel in channels:
|
|
||||||
# slack_channel_id = channel['id']
|
|
||||||
|
|
||||||
# map Slack's topic and purpose content into Zulip's stream description.
|
|
||||||
# WARN This mapping is lossy since the topic.creator, topic.last_set,
|
|
||||||
# purpose.creator, purpose.last_set fields are not preserved.
|
|
||||||
description = "topic: {}\npurpose: {}".format(channel["topic"]["value"],
|
|
||||||
channel["purpose"]["value"])
|
|
||||||
|
|
||||||
# construct the stream object and append it to zerver_stream
|
|
||||||
stream = dict(
|
|
||||||
realm=realm_id,
|
|
||||||
name=channel["name"],
|
|
||||||
deactivated=channel["is_archived"],
|
|
||||||
description=description,
|
|
||||||
invite_only=not channel["is_general"],
|
|
||||||
date_created=float(channel["created"]),
|
|
||||||
id=stream_id_count)
|
|
||||||
|
|
||||||
if channel["name"] == "general":
|
|
||||||
defaultstream = dict(
|
|
||||||
stream=stream_id_count,
|
|
||||||
realm=realm_id,
|
|
||||||
id=1)
|
|
||||||
zerver_defaultstream.append(defaultstream)
|
|
||||||
|
|
||||||
zerver_stream.append(stream)
|
|
||||||
added_channels[stream['name']] = stream_id_count
|
|
||||||
|
|
||||||
# construct the recipient object and append it to zerver_recipient
|
|
||||||
# type 1: private
|
|
||||||
# type 2: stream
|
|
||||||
# type 3: huddle
|
|
||||||
recipient = dict(
|
|
||||||
type_id=stream_id_count,
|
|
||||||
id=stream_id_count,
|
|
||||||
type=2)
|
|
||||||
zerver_recipient.append(recipient)
|
|
||||||
# TOODO add recipients for private message and huddles
|
|
||||||
|
|
||||||
# construct the subscription object and append it to zerver_subscription
|
|
||||||
for member in channel['members']:
|
|
||||||
sub = dict(
|
|
||||||
recipient=stream_id_count,
|
|
||||||
notifications=False,
|
|
||||||
color="#c2c2c2",
|
|
||||||
desktop_notifications=True,
|
|
||||||
pin_to_top=False,
|
|
||||||
in_home_view=True,
|
|
||||||
active=True,
|
|
||||||
user_profile=added_users[member],
|
|
||||||
id=subscription_id_count)
|
|
||||||
# The recipient is a stream for stream-readable message.
|
|
||||||
# proof : https://github.com/zulip/zulip/blob/master/zerver/views/messages.py#L240 &
|
|
||||||
# https://github.com/zulip/zulip/blob/master/zerver/views/messages.py#L324
|
|
||||||
zerver_subscription.append(sub)
|
|
||||||
subscription_id_count += 1
|
|
||||||
# TOODO add zerver_subscription which correspond to
|
|
||||||
# huddles type recipient
|
|
||||||
# For huddles:
|
|
||||||
# sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member]
|
|
||||||
|
|
||||||
# TOODO do private message subscriptions between each users have to
|
|
||||||
# be generated from scratch?
|
|
||||||
|
|
||||||
stream_id_count += 1
|
|
||||||
print(u"{} -> created\n".format(channel['name']))
|
|
||||||
|
|
||||||
# TODO map Slack's pins to Zulip's stars
|
|
||||||
# There is the security model that Slack's pins are known to the team owner
|
|
||||||
# as evident from where it is stored at (channels)
|
|
||||||
# "pins": [
|
|
||||||
# {
|
|
||||||
# "id": "1444755381.000003",
|
|
||||||
# "type": "C",
|
|
||||||
# "user": "U061A5N1G",
|
|
||||||
# "owner": "U061A5N1G",
|
|
||||||
# "created": "1444755463"
|
|
||||||
# }
|
|
||||||
# ],
|
|
||||||
recipient_id_count = stream_id_count + 1
|
|
||||||
subscription_id_count += 1
|
|
||||||
|
|
||||||
for user in zerver_userprofile:
|
|
||||||
zulip_user_id = user['id']
|
|
||||||
|
|
||||||
# this maps the recipients and subscriptions
|
|
||||||
# related to private messages
|
|
||||||
|
|
||||||
recipient = dict(
|
|
||||||
type_id=zulip_user_id,
|
|
||||||
id=recipient_id_count,
|
|
||||||
type=1)
|
|
||||||
zerver_recipient.append(recipient)
|
|
||||||
|
|
||||||
sub = dict(
|
|
||||||
recipient=recipient_id_count,
|
|
||||||
notifications=False,
|
|
||||||
color="#c2c2c2",
|
|
||||||
desktop_notifications=True,
|
|
||||||
pin_to_top=False,
|
|
||||||
in_home_view=True,
|
|
||||||
active=True,
|
|
||||||
user_profile=zulip_user_id,
|
|
||||||
id=subscription_id_count)
|
|
||||||
|
|
||||||
zerver_subscription.append(sub)
|
|
||||||
subscription_id_count += 1
|
|
||||||
recipient_id_count += 1
|
|
||||||
|
|
||||||
print('######### IMPORTING STREAMS FINISHED #########\n')
|
|
||||||
return zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, zerver_recipient
|
|
||||||
|
|
||||||
def main(slack_zip_file: str) -> None:
|
|
||||||
slack_dir = slack_zip_file.replace('.zip', '')
|
|
||||||
subprocess.check_call(['unzip', slack_zip_file])
|
|
||||||
# with zipfile.ZipFile(slack_zip_file, 'r') as zip_ref:
|
|
||||||
# zip_ref.extractall(slack_dir)
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
# TODO fetch realm config from zulip config
|
|
||||||
DOMAIN_NAME = "zulipchat.com"
|
|
||||||
|
|
||||||
# Hardcode this to 1, will implement later for zulipchat.com's case where it
|
|
||||||
# has multiple realms
|
|
||||||
REALM_ID = 1
|
|
||||||
REALM_NAME = "FleshEatingBatswithFangs"
|
|
||||||
NOW = float(datetime.utcnow().timestamp())
|
|
||||||
|
|
||||||
script_path = os.path.dirname(os.path.abspath(__file__)) + '/'
|
|
||||||
zerver_realm_skeleton = json.load(open(script_path + 'zerver_realm_skeleton.json'))
|
|
||||||
zerver_realm_skeleton[0]['id'] = REALM_ID
|
|
||||||
zerver_realm_skeleton[0]['string_id'] = 'zulip' # subdomain / short_name of realm
|
|
||||||
zerver_realm_skeleton[0]['name'] = REALM_NAME
|
|
||||||
zerver_realm_skeleton[0]['date_created'] = NOW
|
|
||||||
|
|
||||||
# Make sure the directory output is clean
|
|
||||||
output_dir = 'zulip_data'
|
|
||||||
rm_tree(output_dir)
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
|
|
||||||
realm = dict(zerver_client=[{"name": "populate_db", "id": 1},
|
|
||||||
{"name": "website", "id": 2},
|
|
||||||
{"name": "API", "id": 3}],
|
|
||||||
zerver_userpresence=[], # shows last logged in data, which is not available in slack
|
|
||||||
zerver_userprofile_mirrordummy=[],
|
|
||||||
zerver_realmdomain=[{"realm": REALM_ID,
|
|
||||||
"allow_subdomains": False,
|
|
||||||
"domain": DOMAIN_NAME,
|
|
||||||
"id": REALM_ID}],
|
|
||||||
zerver_useractivity=[],
|
|
||||||
zerver_realm=zerver_realm_skeleton,
|
|
||||||
zerver_huddle=[],
|
|
||||||
zerver_userprofile_crossrealm=[],
|
|
||||||
zerver_useractivityinterval=[],
|
|
||||||
zerver_realmfilter=[],
|
|
||||||
zerver_realmemoji=[])
|
|
||||||
|
|
||||||
zerver_userprofile, added_users = users2zerver_userprofile(slack_dir,
|
|
||||||
REALM_ID,
|
|
||||||
int(NOW),
|
|
||||||
DOMAIN_NAME)
|
|
||||||
realm['zerver_userprofile'] = zerver_userprofile
|
|
||||||
|
|
||||||
zerver_defaultstream, zerver_stream, added_channels, zerver_subscription, zerver_recipient = channels2zerver_stream(slack_dir,
|
|
||||||
REALM_ID,
|
|
||||||
added_users,
|
|
||||||
zerver_userprofile)
|
|
||||||
# See https://zulipchat.com/help/set-default-streams-for-new-users
|
|
||||||
# for documentation on zerver_defaultstream
|
|
||||||
realm['zerver_defaultstream'] = zerver_defaultstream
|
|
||||||
realm['zerver_stream'] = zerver_stream
|
|
||||||
realm['zerver_subscription'] = zerver_subscription
|
|
||||||
realm['zerver_recipient'] = zerver_recipient
|
|
||||||
# IO realm.json
|
|
||||||
realm_file = output_dir + '/realm.json'
|
|
||||||
json.dump(realm, open(realm_file, 'w'))
|
|
||||||
|
|
||||||
# now for message.json
|
|
||||||
message_json = {}
|
|
||||||
zerver_message = [] # type: List[ZerverFieldsT]
|
|
||||||
zerver_usermessage = [] # type: List[ZerverFieldsT]
|
|
||||||
zerver_attachment = [] # type: List[ZerverFieldsT]
|
|
||||||
|
|
||||||
message_json['zerver_message'] = zerver_message
|
|
||||||
message_json['zerver_usermessage'] = zerver_usermessage
|
|
||||||
# IO message.json
|
|
||||||
message_file = output_dir + '/message.json'
|
|
||||||
json.dump(message_json, open(message_file, 'w'))
|
|
||||||
|
|
||||||
# IO avatar records
|
|
||||||
avatar_records_file = output_dir + '/avatars/records.json'
|
|
||||||
os.makedirs(output_dir + '/avatars', exist_ok=True)
|
|
||||||
json.dump([], open(avatar_records_file, 'w'))
|
|
||||||
|
|
||||||
# IO uploads TODO
|
|
||||||
uploads_records_file = output_dir + '/uploads/records.json'
|
|
||||||
os.makedirs(output_dir + '/uploads', exist_ok=True)
|
|
||||||
json.dump([], open(uploads_records_file, 'w'))
|
|
||||||
|
|
||||||
# IO attachments
|
|
||||||
attachment_file = output_dir + '/attachment.json'
|
|
||||||
attachment = {"zerver_attachment": zerver_attachment}
|
|
||||||
json.dump(attachment, open(attachment_file, 'w'))
|
|
||||||
|
|
||||||
print('ls', os.listdir())
|
|
||||||
print('pwd', os.getcwd())
|
|
||||||
|
|
||||||
# remove slack dir
|
|
||||||
rm_tree(slack_dir)
|
|
||||||
|
|
||||||
# compress the folder
|
|
||||||
subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir])
|
|
||||||
# subprocess.check_call(['zip', '-jpr', output_dir + '.zip', realm_file, message_file])
|
|
||||||
|
|
||||||
# remove zulip dir
|
|
||||||
rm_tree(output_dir)
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
# from django.conf import settings
|
|
||||||
# settings_module = "settings.py"
|
|
||||||
# os.environ['DJANGO_SETTINGS_MODULE'] = settings_module
|
|
||||||
description = ("script to convert Slack export data into Zulip export data")
|
|
||||||
parser = argparse.ArgumentParser(description=description)
|
|
||||||
slack_zip_file = sys.argv[1]
|
|
||||||
main(slack_zip_file)
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
[{
|
|
||||||
"message_retention_days": null,
|
|
||||||
"inline_image_preview": true,
|
|
||||||
"name_changes_disabled": false,
|
|
||||||
"string_id": "zulip",
|
|
||||||
"icon_version": 1,
|
|
||||||
"waiting_period_threshold": 0,
|
|
||||||
"email_changes_disabled": false,
|
|
||||||
"deactivated": false,
|
|
||||||
"notifications_stream": null,
|
|
||||||
"restricted_to_domain": true,
|
|
||||||
"show_digest_email": true,
|
|
||||||
"allow_message_editing": true,
|
|
||||||
"description": "The Zulip development environment default organization. It's great for testing!",
|
|
||||||
"default_language": "en",
|
|
||||||
"icon_source": "G",
|
|
||||||
"invite_required": false,
|
|
||||||
"invite_by_admins_only": false,
|
|
||||||
"create_stream_by_admins_only": false,
|
|
||||||
"mandatory_topics": false,
|
|
||||||
"inline_url_embed_preview": true,
|
|
||||||
"message_content_edit_limit_seconds": 600,
|
|
||||||
"authentication_methods": [
|
|
||||||
["Google", true],
|
|
||||||
["Email", true],
|
|
||||||
["GitHub", true],
|
|
||||||
["LDAP", true],
|
|
||||||
["Dev", true],
|
|
||||||
["RemoteUser", true]
|
|
||||||
],
|
|
||||||
"name": "",
|
|
||||||
"org_type": 1,
|
|
||||||
"add_emoji_by_admins_only": false,
|
|
||||||
"date_created": 0.0,
|
|
||||||
"id": 1
|
|
||||||
}]
|
|
||||||
Loading…
Reference in a new issue