314 lines
12 KiB
Python
Executable file
314 lines
12 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# Zulip mirror of Codebase HQ activity
|
|
# The "zulip_codebase_mirror" script is run continuously, possibly on a work
|
|
# computer or preferably on a server.
|
|
#
|
|
# When restarted, it will attempt to pick up where it left off.
|
|
#
|
|
# python-dateutil is a dependency for this script.
|
|
|
|
from __future__ import print_function
|
|
from __future__ import absolute_import
|
|
import requests
|
|
import logging
|
|
import pytz
|
|
import time
|
|
import sys
|
|
import os
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
try:
|
|
import dateutil.parser
|
|
except ImportError as e:
|
|
print(e, file=sys.stderr)
|
|
print("Please install the python-dateutil package.", file=sys.stderr)
|
|
exit(1)
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
import zulip_codebase_config as config
|
|
VERSION = "0.9"
|
|
|
|
if config.ZULIP_API_PATH is not None:
|
|
sys.path.append(config.ZULIP_API_PATH)
|
|
import six
|
|
import zulip
|
|
from typing import Any, List, Dict, Optional
|
|
|
|
client = zulip.Client(
|
|
email=config.ZULIP_USER,
|
|
site=config.ZULIP_SITE,
|
|
api_key=config.ZULIP_API_KEY,
|
|
client="ZulipCodebase/" + VERSION)
|
|
user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
|
|
|
|
# find some form of JSON loader/dumper, with a preference order for speed.
|
|
json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
|
|
|
|
while len(json_implementations):
|
|
try:
|
|
json = __import__(json_implementations.pop(0))
|
|
break
|
|
except ImportError:
|
|
continue
|
|
|
|
def make_api_call(path):
|
|
# type: (str) -> Optional[List[Dict[str, Any]]]
|
|
response = requests.get("https://api3.codebasehq.com/%s" % (path,),
|
|
auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
|
|
params={'raw': 'True'},
|
|
headers = {"User-Agent": user_agent,
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json"})
|
|
if response.status_code == 200:
|
|
return json.loads(response.text)
|
|
|
|
if response.status_code >= 500:
|
|
logging.error(str(response.status_code))
|
|
return None
|
|
if response.status_code == 403:
|
|
logging.error("Bad authorization from Codebase. Please check your credentials")
|
|
sys.exit(-1)
|
|
else:
|
|
logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
|
|
return None
|
|
|
|
def make_url(path):
|
|
# type: (str) -> str
|
|
return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
|
|
|
|
def handle_event(event):
|
|
# type: (Dict[str, Any]) -> None
|
|
event = event['event']
|
|
event_type = event['type']
|
|
actor_name = event['actor_name']
|
|
|
|
raw_props = event.get('raw_properties', {})
|
|
|
|
project_link = raw_props.get('project_permalink')
|
|
|
|
subject = None
|
|
content = None
|
|
if event_type == 'repository_creation':
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
project_name = raw_props.get('name')
|
|
project_repo_type = raw_props.get('scm_type')
|
|
|
|
url = make_url("projects/%s" % (project_link,))
|
|
scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
|
|
|
|
subject = "Repository %s Created" % (project_name,)
|
|
content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
|
|
elif event_type == 'push':
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
num_commits = raw_props.get('commits_count')
|
|
branch = raw_props.get('ref_name')
|
|
project = raw_props.get('project_name')
|
|
repo_link = raw_props.get('repository_permalink')
|
|
deleted_ref = raw_props.get('deleted_ref')
|
|
new_ref = raw_props.get('new_ref')
|
|
|
|
subject = "Push to %s on %s" % (branch, project)
|
|
|
|
if deleted_ref:
|
|
content = "%s deleted branch %s from %s" % (actor_name, branch, project)
|
|
else:
|
|
if new_ref:
|
|
branch = "new branch %s" % (branch,)
|
|
content = ("%s pushed %s commit(s) to %s in project %s:\n\n" %
|
|
(actor_name, num_commits, branch, project))
|
|
for commit in raw_props.get('commits'):
|
|
ref = commit.get('ref')
|
|
url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
|
|
message = commit.get('message')
|
|
content += "* [%s](%s): %s\n" % (ref, url, message)
|
|
elif event_type == 'ticketing_ticket':
|
|
stream = config.ZULIP_TICKETS_STREAM_NAME
|
|
|
|
num = raw_props.get('number')
|
|
name = raw_props.get('subject')
|
|
assignee = raw_props.get('assignee')
|
|
priority = raw_props.get('priority')
|
|
url = make_url("projects/%s/tickets/%s" % (project_link, num))
|
|
|
|
if assignee is None:
|
|
assignee = "no one"
|
|
subject = "#%s: %s" % (num, name)
|
|
content = ("""%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" %
|
|
(actor_name, num, url, priority, assignee, name))
|
|
elif event_type == 'ticketing_note':
|
|
stream = config.ZULIP_TICKETS_STREAM_NAME
|
|
|
|
num = raw_props.get('number')
|
|
name = raw_props.get('subject')
|
|
body = raw_props.get('content')
|
|
changes = raw_props.get('changes')
|
|
|
|
url = make_url("projects/%s/tickets/%s" % (project_link, num))
|
|
subject = "#%s: %s" % (num, name)
|
|
|
|
content = ""
|
|
if body is not None and len(body) > 0:
|
|
content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
|
|
|
|
if 'status_id' in changes:
|
|
status_change = changes.get('status_id')
|
|
content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
|
|
elif event_type == 'ticketing_milestone':
|
|
stream = config.ZULIP_TICKETS_STREAM_NAME
|
|
|
|
name = raw_props.get('name')
|
|
identifier = raw_props.get('identifier')
|
|
url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
|
|
|
|
subject = name
|
|
content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
|
|
elif event_type == 'comment':
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
comment = raw_props.get('content')
|
|
commit = raw_props.get('commit_ref')
|
|
|
|
# If there's a commit id, it's a comment to a commit
|
|
if commit:
|
|
repo_link = raw_props.get('repository_permalink')
|
|
|
|
url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
|
|
|
|
subject = "%s commented on %s" % (actor_name, commit)
|
|
content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
|
|
else:
|
|
# Otherwise, this is a Discussion item, and handle it
|
|
subj = raw_props.get("subject")
|
|
category = raw_props.get("category")
|
|
comment_content = raw_props.get("content")
|
|
|
|
subject = "Discussion: %s" % (subj,)
|
|
|
|
if category:
|
|
format_str = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~"
|
|
content = format_str % (actor_name, category, comment_content)
|
|
else:
|
|
content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
|
|
|
|
elif event_type == 'deployment':
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
start_ref = raw_props.get('start_ref')
|
|
end_ref = raw_props.get('end_ref')
|
|
environment = raw_props.get('environment')
|
|
servers = raw_props.get('servers')
|
|
repo_link = raw_props.get('repository_permalink')
|
|
|
|
start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
|
|
end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
|
|
between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (
|
|
project_link, repo_link, start_ref, end_ref))
|
|
|
|
subject = "Deployment to %s" % (environment,)
|
|
|
|
content = ("%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." %
|
|
(actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment))
|
|
if servers is not None:
|
|
content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
|
|
|
|
elif event_type == 'named_tree':
|
|
# Docs say named_tree type used for new/deleting branches and tags,
|
|
# but experimental testing showed that they were all sent as 'push' events
|
|
pass
|
|
elif event_type == 'wiki_page':
|
|
logging.warn("Wiki page notifications not yet implemented")
|
|
elif event_type == 'sprint_creation':
|
|
logging.warn("Sprint notifications not yet implemented")
|
|
elif event_type == 'sprint_ended':
|
|
logging.warn("Sprint notifications not yet implemented")
|
|
else:
|
|
logging.info("Unknown event type %s, ignoring!" % (event_type,))
|
|
|
|
if subject and content:
|
|
if len(subject) > 60:
|
|
subject = subject[:57].rstrip() + '...'
|
|
|
|
res = client.send_message({"type": "stream",
|
|
"to": stream,
|
|
"subject": subject,
|
|
"content": content})
|
|
if res['result'] == 'success':
|
|
logging.info("Successfully sent Zulip with id: %s" % (res['id']))
|
|
else:
|
|
logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg']))
|
|
|
|
|
|
# the main run loop for this mirror script
|
|
def run_mirror():
|
|
# type: () -> None
|
|
# we should have the right (write) permissions on the resume file, as seen
|
|
# in check_permissions, but it may still be empty or corrupted
|
|
def default_since():
|
|
# type: () -> datetime
|
|
return datetime.now(tz=pytz.utc) - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
|
|
|
|
try:
|
|
with open(config.RESUME_FILE) as f:
|
|
timestamp = f.read()
|
|
if timestamp == '':
|
|
since = default_since()
|
|
else:
|
|
since = datetime.fromtimestamp(float(timestamp), tz=pytz.utc)
|
|
except (ValueError, IOError) as e:
|
|
logging.warn("Could not open resume file: %s" % (str(e)))
|
|
since = default_since()
|
|
|
|
try:
|
|
sleepInterval = 1
|
|
while True:
|
|
events = make_api_call("activity")
|
|
if events is not None:
|
|
sleepInterval = 1
|
|
for event in events[::-1]:
|
|
timestamp = event.get('event', {}).get('timestamp', '')
|
|
event_date = dateutil.parser.parse(timestamp)
|
|
if event_date > since:
|
|
handle_event(event)
|
|
since = event_date
|
|
else:
|
|
# back off a bit
|
|
if sleepInterval < 22:
|
|
sleepInterval += 4
|
|
time.sleep(sleepInterval)
|
|
|
|
except KeyboardInterrupt:
|
|
open(config.RESUME_FILE, 'w').write(since.strftime("%s"))
|
|
logging.info("Shutting down Codebase mirror")
|
|
|
|
# void function that checks the permissions of the files this script needs.
|
|
def check_permissions():
|
|
# type: () -> None
|
|
# check that the log file can be written
|
|
if config.LOG_FILE:
|
|
try:
|
|
open(config.LOG_FILE, "w")
|
|
except IOError as e:
|
|
sys.stderr.write("Could not open up log for writing:")
|
|
sys.stderr.write(str(e))
|
|
# check that the resume file can be written (this creates if it doesn't exist)
|
|
try:
|
|
open(config.RESUME_FILE, "a+")
|
|
except IOError as e:
|
|
sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
|
|
sys.stderr.write(str(e))
|
|
|
|
if __name__ == "__main__":
|
|
if not isinstance(config.RESUME_FILE, six.string_types):
|
|
sys.stderr.write("RESUME_FILE path not given; refusing to continue")
|
|
check_permissions()
|
|
if config.LOG_FILE:
|
|
logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
|
|
else:
|
|
logging.basicConfig(level=logging.WARNING)
|
|
run_mirror()
|