186 lines
7.6 KiB
Python
Executable file
186 lines
7.6 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Zulip mirror of Basecamp activity
|
|
# Copyright © 2014 Zulip, Inc.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
# THE SOFTWARE.
|
|
#
|
|
# The "basecamp-mirror.py" script is run continuously, possibly on a work computer
|
|
# or preferably on a server.
|
|
# You may need to install the python-requests library.
|
|
|
|
from __future__ import absolute_import
|
|
import requests
|
|
import logging
|
|
import time
|
|
import re
|
|
import sys
|
|
from stderror import write
|
|
import os
|
|
from datetime import datetime, timedelta
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
import zulip_basecamp_config as config
|
|
VERSION = "0.9"
|
|
|
|
if config.ZULIP_API_PATH is not None:
|
|
sys.path.append(config.ZULIP_API_PATH)
|
|
import zulip
|
|
from six.moves.html_parser import HTMLParser
|
|
from typing import Any, Dict
|
|
import six
|
|
|
|
client = zulip.Client(
|
|
email=config.ZULIP_USER,
|
|
site=config.ZULIP_SITE,
|
|
api_key=config.ZULIP_API_KEY,
|
|
client="ZulipBasecamp/" + VERSION)
|
|
user_agent = "Basecamp To Zulip Mirroring script (zulip-devel@googlegroups.com)"
|
|
htmlParser = HTMLParser()
|
|
|
|
# find some form of JSON loader/dumper, with a preference order for speed.
|
|
json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
|
|
|
|
while len(json_implementations):
|
|
try:
|
|
json = __import__(json_implementations.pop(0))
|
|
break
|
|
except ImportError:
|
|
continue
|
|
|
|
# void function that checks the permissions of the files this script needs.
|
|
def check_permissions():
|
|
# type: () -> None
|
|
# check that the log file can be written
|
|
if config.LOG_FILE:
|
|
try:
|
|
open(config.LOG_FILE, "w")
|
|
except IOError as e:
|
|
sys.stderr.write("Could not open up log for writing:")
|
|
sys.stderr.write(str(e))
|
|
# check that the resume file can be written (this creates if it doesn't exist)
|
|
try:
|
|
open(config.RESUME_FILE, "a+")
|
|
except IOError as e:
|
|
sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE),)
|
|
sys.stderr.write(str(e))
|
|
|
|
# builds the message dict for sending a message with the Zulip API
|
|
def build_message(event):
|
|
# type: (Dict[str, Any]) -> Dict[str, Any]
|
|
if not ('bucket' in event and 'creator' in event and 'html_url' in event):
|
|
logging.error("Perhaps the Basecamp API changed behavior? "
|
|
"This event doesn't have the expected format:\n%s" % (event,))
|
|
return None
|
|
# adjust the topic length to be bounded to 60 characters
|
|
topic = event['bucket']['name']
|
|
if len(topic) > 60:
|
|
topic = topic[0:57] + "..."
|
|
# get the action and target values
|
|
action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
|
|
target = htmlParser.unescape(event.get('target', ''))
|
|
# Some events have "excerpts", which we blockquote
|
|
excerpt = htmlParser.unescape(event.get('excerpt', ''))
|
|
if excerpt.strip() == "":
|
|
message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
|
|
else:
|
|
message = '**%s** %s [%s](%s).\n> %s' % (event['creator']['name'], action, target, event['html_url'], excerpt)
|
|
# assemble the message data dict
|
|
message_data = {
|
|
"type": "stream",
|
|
"to": config.ZULIP_STREAM_NAME,
|
|
"subject": topic,
|
|
"content": message,
|
|
}
|
|
return message_data
|
|
|
|
# the main run loop for this mirror script
|
|
def run_mirror():
|
|
# type: () -> None
|
|
# we should have the right (write) permissions on the resume file, as seen
|
|
# in check_permissions, but it may still be empty or corrupted
|
|
try:
|
|
with open(config.RESUME_FILE) as f:
|
|
since = f.read() # type: Any
|
|
since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
|
|
assert since, "resume file does not meet expected format"
|
|
since = since.string
|
|
except (AssertionError, IOError) as e:
|
|
logging.warn("Could not open resume file: %s" % (e,))
|
|
since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
|
|
try:
|
|
# we use an exponential backoff approach when we get 429 (Too Many Requests).
|
|
sleepInterval = 1
|
|
while True:
|
|
time.sleep(sleepInterval)
|
|
response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
|
|
params={'since': since},
|
|
auth=(config.BASECAMP_USERNAME, config.BASECAMP_PASSWORD),
|
|
headers = {"User-Agent": user_agent})
|
|
if response.status_code == 200:
|
|
sleepInterval = 1
|
|
events = json.loads(response.text)
|
|
if len(events):
|
|
logging.info("Got event(s): %s" % (response.text,))
|
|
if response.status_code >= 500:
|
|
logging.error(str(response.status_code))
|
|
continue
|
|
if response.status_code == 429:
|
|
# exponential backoff
|
|
sleepInterval *= 2
|
|
logging.error(str(response.status_code))
|
|
continue
|
|
if response.status_code == 400:
|
|
logging.error("Something went wrong. Basecamp must be unhappy for this reason: %s" % (response.text,))
|
|
sys.exit(-1)
|
|
if response.status_code == 401:
|
|
logging.error("Bad authorization from Basecamp. Please check your Basecamp login credentials")
|
|
sys.exit(-1)
|
|
if len(events):
|
|
since = events[0]['created_at']
|
|
for event in reversed(events):
|
|
message_data = build_message(event)
|
|
if not message_data:
|
|
continue
|
|
zulip_api_result = client.send_message(message_data)
|
|
if zulip_api_result['result'] == "success":
|
|
logging.info("sent zulip with id: %s" % (zulip_api_result['id'],))
|
|
else:
|
|
logging.warn("%s %s" % (zulip_api_result['result'], zulip_api_result['msg']))
|
|
# update 'since' each time in case we get KeyboardInterrupted
|
|
since = event['created_at']
|
|
# avoid hitting rate-limit
|
|
time.sleep(0.2)
|
|
|
|
except KeyboardInterrupt:
|
|
logging.info("Shutting down, please hold")
|
|
open("events.last", 'w').write(since)
|
|
logging.info("Done!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if not isinstance(config.RESUME_FILE, six.string_types):
|
|
sys.stderr.write("RESUME_FILE path not given; refusing to continue")
|
|
check_permissions()
|
|
if config.LOG_FILE:
|
|
logging.basicConfig(filename=config.LOG_FILE, level=logging.INFO)
|
|
else:
|
|
logging.basicConfig(level=logging.INFO)
|
|
run_mirror()
|