#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- # # Zulip mirror of Codebase HQ activity # Copyright © 2014 Zulip, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # The "zulip_codebase_mirror" script is run continuously, possibly on a work # computer or preferably on a server. # # When restarted, it will attempt to pick up where it left off. # # python-dateutil is a dependency for this script. import requests import logging import time import sys import os from datetime import datetime, timedelta try: import dateutil.parser except ImportError, e: print >>sys.stderr, e print >>sys.stderr, "Please install the python-dateutil package." exit(1) sys.path.insert(0, os.path.dirname(__file__)) import zulip_codebase_config as config VERSION = "0.9" if config.ZULIP_API_PATH is not None: sys.path.append(config.ZULIP_API_PATH) import zulip client = zulip.Client( email=config.ZULIP_USER, site=config.ZULIP_SITE, api_key=config.ZULIP_API_KEY, client="ZulipCodebase/" + VERSION) user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)" # find some form of JSON loader/dumper, with a preference order for speed. json_implementations = ['ujson', 'cjson', 'simplejson', 'json'] while len(json_implementations): try: json = __import__(json_implementations.pop(0)) break except ImportError: continue def make_api_call(path): response = requests.get("https://api3.codebasehq.com/%s" % (path,), auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY), params={'raw': True}, headers = {"User-Agent": user_agent, "Content-Type": "application/json", "Accept": "application/json"}) if response.status_code == 200: return json.loads(response.text) if response.status_code >= 500: logging.error(response.status_code) return None if response.status_code == 403: logging.error("Bad authorization from Codebase. Please check your credentials") sys.exit(-1) else: logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text)) return None def make_url(path): return "%s/%s" % (config.CODEBASE_ROOT_URL, path) def handle_event(event): event = event['event'] event_type = event['type'] actor_name = event['actor_name'] raw_props = event.get('raw_properties', {}) project_link = raw_props.get('project_permalink') subject = None content = None if event_type == 'repository_creation': stream = config.ZULIP_COMMITS_STREAM_NAME project_name = raw_props.get('name') project_repo_type = raw_props.get('scm_type') url = make_url("projects/%s" % (project_link,)) scm = "of type %s" % (project_repo_type,) if project_repo_type else "" subject = "Repository %s Created" % (project_name,) content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url) elif event_type == 'push': stream = config.ZULIP_COMMITS_STREAM_NAME num_commits = raw_props.get('commits_count') branch = raw_props.get('ref_name') project = raw_props.get('project_name') repo_link = raw_props.get('repository_permalink') deleted_ref = raw_props.get('deleted_ref') new_ref = raw_props.get('new_ref') subject = "Push to %s on %s" % (branch, project) if deleted_ref: content = "%s deleted branch %s from %s" % (actor_name, branch, project) else: if new_ref: branch = "new branch %s" % (branch, ) content = "%s pushed %s commit(s) to %s in project %s:\n\n" % \ (actor_name, num_commits, branch, project) for commit in raw_props.get('commits'): ref = commit.get('ref') url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref)) message = commit.get('message') content += "* [%s](%s): %s\n" % (ref, url, message) elif event_type == 'ticketing_ticket': stream = config.ZULIP_TICKETS_STREAM_NAME num = raw_props.get('number') name = raw_props.get('subject') assignee = raw_props.get('assignee') priority = raw_props.get('priority') url = make_url("projects/%s/tickets/%s" % (project_link, num)) if assignee is None: assignee = "no one" subject = "#%s: %s" % (num, name) content = """%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" % \ (actor_name, num, url, priority, assignee, name) elif event_type == 'ticketing_note': stream = config.ZULIP_TICKETS_STREAM_NAME num = raw_props.get('number') name = raw_props.get('subject') body = raw_props.get('content') changes = raw_props.get('changes') url = make_url("projects/%s/tickets/%s" % (project_link, num)) subject = "#%s: %s" % (num, name) content = "" if body is not None and len(body) > 0: content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body) if 'status_id' in changes: status_change = changes.get('status_id') content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1]) elif event_type == 'ticketing_milestone': stream = config.ZULIP_TICKETS_STREAM_NAME name = raw_props.get('name') identifier = raw_props.get('identifier') url = make_url("projects/%s/milestone/%s" % (project_link, identifier)) subject = name content = "%s created a new milestone [%s](%s)" % (actor_name, name, url) elif event_type == 'comment': stream = config.ZULIP_COMMITS_STREAM_NAME comment = raw_props.get('content') commit = raw_props.get('commit_ref') # If there's a commit id, it's a comment to a commit if commit: repo_link = raw_props.get('repository_permalink') url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit)) subject = "%s commented on %s" % (actor_name, commit) content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment) else: # Otherwise, this is a Discussion item, and handle it subj = raw_props.get("subject") category = raw_props.get("category") comment_content = raw_props.get("content") subject = "Discussion: %s" % (subj,) if category: content = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~" % (actor_name, category, comment_content) else: content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content) elif event_type == 'deployment': stream = config.ZULIP_COMMITS_STREAM_NAME start_ref = raw_props.get('start_ref') end_ref = raw_props.get('end_ref') environment = raw_props.get('environment') servers = raw_props.get('servers') repo_link = raw_props.get('repository_permalink') start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref)) end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref)) between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (project_link, repo_link, start_ref, end_ref)) subject = "Deployment to %s" % (environment,) content = "%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." % \ (actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment) if servers is not None: content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers])) elif event_type == 'named_tree': # Docs say named_tree type used for new/deleting branches and tags, # but experimental testing showed that they were all sent as 'push' events pass elif event_type == 'wiki_page': logging.warn("Wiki page notifications not yet implemented") elif event_type == 'sprint_creation': logging.warn("Sprint notifications not yet implemented") elif event_type == 'sprint_ended': logging.warn("Sprint notifications not yet implemented") else: logging.info("Unknown event type %s, ignoring!" % (event_type,)) if subject and content: if len(subject) > 60: subject = subject[:57].rstrip() + '...' res = client.send_message({"type": "stream", "to": stream, "subject": subject, "content": content}) if res['result'] == 'success': logging.info("Successfully sent Zulip with id: %s" % (res['id'])) else: logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg'])) # the main run loop for this mirror script def run_mirror(): # we should have the right (write) permissions on the resume file, as seen # in check_permissions, but it may still be empty or corrupted def default_since(): return datetime.utcnow() - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS) try: with open(config.RESUME_FILE) as f: timestamp = f.read() if timestamp == '': since = default_since() else: timestamp = int(timestamp, 10) since = datetime.fromtimestamp(timestamp) except (ValueError,IOError) as e: logging.warn("Could not open resume file: %s" % (e.message or e.strerror,)) since = default_since() try: sleepInterval = 1 while 1: events = make_api_call("activity")[::-1] if events is not None: sleepInterval = 1 for event in events: timestamp = event.get('event', {}).get('timestamp', '') event_date = dateutil.parser.parse(timestamp).replace(tzinfo=None) if event_date > since: handle_event(event) since = event_date else: # back off a bit if sleepInterval < 22: sleepInterval += 4 time.sleep(sleepInterval) except KeyboardInterrupt: open(config.RESUME_FILE, 'w').write(since.strftime("%s")); logging.info("Shutting down Codebase mirror") # void function that checks the permissions of the files this script needs. def check_permissions(): # check that the log file can be written if config.LOG_FILE: try: open(config.LOG_FILE, "w") except IOError as e: sys.stderr("Could not open up log for writing:") sys.stderr(e) # check that the resume file can be written (this creates if it doesn't exist) try: open(config.RESUME_FILE, "a+") except IOError as e: sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,)) sys.stderr(e) if __name__ == "__main__": if not isinstance(config.RESUME_FILE, basestring): sys.stderr("RESUME_FILE path not given; refusing to continue") check_permissions() if config.LOG_FILE: logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING) else: logging.basicConfig(level=logging.WARNING) run_mirror()