356 lines
12 KiB
Python
Executable file
356 lines
12 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
# Zulip mirror of Codebase HQ activity
|
|
# The "zulip_codebase_mirror" script is run continuously, possibly on a work
|
|
# computer or preferably on a server.
|
|
#
|
|
# When restarted, it will attempt to pick up where it left off.
|
|
#
|
|
# python-dateutil is a dependency for this script.
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
|
|
import pytz
|
|
import requests
|
|
|
|
try:
|
|
import dateutil.parser
|
|
except ImportError as e:
|
|
print(e, file=sys.stderr)
|
|
print("Please install the python-dateutil package.", file=sys.stderr)
|
|
exit(1)
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
import zulip_codebase_config as config
|
|
|
|
VERSION = "0.9"
|
|
|
|
if config.ZULIP_API_PATH is not None:
|
|
sys.path.append(config.ZULIP_API_PATH)
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import zulip
|
|
|
|
client = zulip.Client(
|
|
email=config.ZULIP_USER,
|
|
site=config.ZULIP_SITE,
|
|
api_key=config.ZULIP_API_KEY,
|
|
client="ZulipCodebase/" + VERSION,
|
|
)
|
|
user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
|
|
|
|
# find some form of JSON loader/dumper, with a preference order for speed.
|
|
json_implementations = ["ujson", "cjson", "simplejson", "json"]
|
|
|
|
while len(json_implementations):
|
|
try:
|
|
json = __import__(json_implementations.pop(0))
|
|
break
|
|
except ImportError:
|
|
continue
|
|
|
|
|
|
def make_api_call(path: str) -> Optional[List[Dict[str, Any]]]:
|
|
response = requests.get(
|
|
"https://api3.codebasehq.com/%s" % (path,),
|
|
auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
|
|
params={"raw": "True"},
|
|
headers={
|
|
"User-Agent": user_agent,
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json",
|
|
},
|
|
)
|
|
if response.status_code == 200:
|
|
return json.loads(response.text)
|
|
|
|
if response.status_code >= 500:
|
|
logging.error(str(response.status_code))
|
|
return None
|
|
if response.status_code == 403:
|
|
logging.error("Bad authorization from Codebase. Please check your credentials")
|
|
sys.exit(-1)
|
|
else:
|
|
logging.warn(
|
|
"Found non-success response status code: %s %s" % (response.status_code, response.text)
|
|
)
|
|
return None
|
|
|
|
|
|
def make_url(path: str) -> str:
|
|
return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
|
|
|
|
|
|
def handle_event(event: Dict[str, Any]) -> None:
|
|
event = event["event"]
|
|
event_type = event["type"]
|
|
actor_name = event["actor_name"]
|
|
|
|
raw_props = event.get("raw_properties", {})
|
|
|
|
project_link = raw_props.get("project_permalink")
|
|
|
|
subject = None
|
|
content = None
|
|
if event_type == "repository_creation":
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
project_name = raw_props.get("name")
|
|
project_repo_type = raw_props.get("scm_type")
|
|
|
|
url = make_url("projects/%s" % (project_link,))
|
|
scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
|
|
|
|
subject = "Repository %s Created" % (project_name,)
|
|
content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
|
|
elif event_type == "push":
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
num_commits = raw_props.get("commits_count")
|
|
branch = raw_props.get("ref_name")
|
|
project = raw_props.get("project_name")
|
|
repo_link = raw_props.get("repository_permalink")
|
|
deleted_ref = raw_props.get("deleted_ref")
|
|
new_ref = raw_props.get("new_ref")
|
|
|
|
subject = "Push to %s on %s" % (branch, project)
|
|
|
|
if deleted_ref:
|
|
content = "%s deleted branch %s from %s" % (actor_name, branch, project)
|
|
else:
|
|
if new_ref:
|
|
branch = "new branch %s" % (branch,)
|
|
content = "%s pushed %s commit(s) to %s in project %s:\n\n" % (
|
|
actor_name,
|
|
num_commits,
|
|
branch,
|
|
project,
|
|
)
|
|
for commit in raw_props.get("commits"):
|
|
ref = commit.get("ref")
|
|
url = make_url(
|
|
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref)
|
|
)
|
|
message = commit.get("message")
|
|
content += "* [%s](%s): %s\n" % (ref, url, message)
|
|
elif event_type == "ticketing_ticket":
|
|
stream = config.ZULIP_TICKETS_STREAM_NAME
|
|
|
|
num = raw_props.get("number")
|
|
name = raw_props.get("subject")
|
|
assignee = raw_props.get("assignee")
|
|
priority = raw_props.get("priority")
|
|
url = make_url("projects/%s/tickets/%s" % (project_link, num))
|
|
|
|
if assignee is None:
|
|
assignee = "no one"
|
|
subject = "#%s: %s" % (num, name)
|
|
content = (
|
|
"""%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s"""
|
|
% (actor_name, num, url, priority, assignee, name)
|
|
)
|
|
elif event_type == "ticketing_note":
|
|
stream = config.ZULIP_TICKETS_STREAM_NAME
|
|
|
|
num = raw_props.get("number")
|
|
name = raw_props.get("subject")
|
|
body = raw_props.get("content")
|
|
changes = raw_props.get("changes")
|
|
|
|
url = make_url("projects/%s/tickets/%s" % (project_link, num))
|
|
subject = "#%s: %s" % (num, name)
|
|
|
|
content = ""
|
|
if body is not None and len(body) > 0:
|
|
content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (
|
|
actor_name,
|
|
num,
|
|
url,
|
|
body,
|
|
)
|
|
|
|
if "status_id" in changes:
|
|
status_change = changes.get("status_id")
|
|
content += "Status changed from **%s** to **%s**\n\n" % (
|
|
status_change[0],
|
|
status_change[1],
|
|
)
|
|
elif event_type == "ticketing_milestone":
|
|
stream = config.ZULIP_TICKETS_STREAM_NAME
|
|
|
|
name = raw_props.get("name")
|
|
identifier = raw_props.get("identifier")
|
|
url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
|
|
|
|
subject = name
|
|
content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
|
|
elif event_type == "comment":
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
comment = raw_props.get("content")
|
|
commit = raw_props.get("commit_ref")
|
|
|
|
# If there's a commit id, it's a comment to a commit
|
|
if commit:
|
|
repo_link = raw_props.get("repository_permalink")
|
|
|
|
url = make_url(
|
|
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, commit)
|
|
)
|
|
|
|
subject = "%s commented on %s" % (actor_name, commit)
|
|
content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (
|
|
actor_name,
|
|
commit,
|
|
url,
|
|
comment,
|
|
)
|
|
else:
|
|
# Otherwise, this is a Discussion item, and handle it
|
|
subj = raw_props.get("subject")
|
|
category = raw_props.get("category")
|
|
comment_content = raw_props.get("content")
|
|
|
|
subject = "Discussion: %s" % (subj,)
|
|
|
|
if category:
|
|
format_str = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~"
|
|
content = format_str % (actor_name, category, comment_content)
|
|
else:
|
|
content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
|
|
|
|
elif event_type == "deployment":
|
|
stream = config.ZULIP_COMMITS_STREAM_NAME
|
|
|
|
start_ref = raw_props.get("start_ref")
|
|
end_ref = raw_props.get("end_ref")
|
|
environment = raw_props.get("environment")
|
|
servers = raw_props.get("servers")
|
|
repo_link = raw_props.get("repository_permalink")
|
|
|
|
start_ref_url = make_url(
|
|
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref)
|
|
)
|
|
end_ref_url = make_url(
|
|
"projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref)
|
|
)
|
|
between_url = make_url(
|
|
"projects/%s/repositories/%s/compare/%s...%s"
|
|
% (project_link, repo_link, start_ref, end_ref)
|
|
)
|
|
|
|
subject = "Deployment to %s" % (environment,)
|
|
|
|
content = "%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." % (
|
|
actor_name,
|
|
start_ref,
|
|
start_ref_url,
|
|
between_url,
|
|
end_ref,
|
|
end_ref_url,
|
|
environment,
|
|
)
|
|
if servers is not None:
|
|
content += "\n\nServers deployed to: %s" % (
|
|
", ".join(["`%s`" % (server,) for server in servers])
|
|
)
|
|
|
|
elif event_type == "named_tree":
|
|
# Docs say named_tree type used for new/deleting branches and tags,
|
|
# but experimental testing showed that they were all sent as 'push' events
|
|
pass
|
|
elif event_type == "wiki_page":
|
|
logging.warn("Wiki page notifications not yet implemented")
|
|
elif event_type == "sprint_creation":
|
|
logging.warn("Sprint notifications not yet implemented")
|
|
elif event_type == "sprint_ended":
|
|
logging.warn("Sprint notifications not yet implemented")
|
|
else:
|
|
logging.info("Unknown event type %s, ignoring!" % (event_type,))
|
|
|
|
if subject and content:
|
|
if len(subject) > 60:
|
|
subject = subject[:57].rstrip() + "..."
|
|
|
|
res = client.send_message(
|
|
{"type": "stream", "to": stream, "subject": subject, "content": content}
|
|
)
|
|
if res["result"] == "success":
|
|
logging.info("Successfully sent Zulip with id: %s" % (res["id"],))
|
|
else:
|
|
logging.warn("Failed to send Zulip: %s %s" % (res["result"], res["msg"]))
|
|
|
|
|
|
# the main run loop for this mirror script
|
|
def run_mirror() -> None:
|
|
# we should have the right (write) permissions on the resume file, as seen
|
|
# in check_permissions, but it may still be empty or corrupted
|
|
def default_since() -> datetime:
|
|
return datetime.now(tz=pytz.utc) - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
|
|
|
|
try:
|
|
with open(config.RESUME_FILE) as f:
|
|
timestamp = f.read()
|
|
if timestamp == "":
|
|
since = default_since()
|
|
else:
|
|
since = datetime.fromtimestamp(float(timestamp), tz=pytz.utc)
|
|
except (ValueError, OSError) as e:
|
|
logging.warn("Could not open resume file: %s" % (str(e),))
|
|
since = default_since()
|
|
|
|
try:
|
|
sleepInterval = 1
|
|
while True:
|
|
events = make_api_call("activity")
|
|
if events is not None:
|
|
sleepInterval = 1
|
|
for event in events[::-1]:
|
|
timestamp = event.get("event", {}).get("timestamp", "")
|
|
event_date = dateutil.parser.parse(timestamp)
|
|
if event_date > since:
|
|
handle_event(event)
|
|
since = event_date
|
|
else:
|
|
# back off a bit
|
|
if sleepInterval < 22:
|
|
sleepInterval += 4
|
|
time.sleep(sleepInterval)
|
|
|
|
except KeyboardInterrupt:
|
|
open(config.RESUME_FILE, "w").write(since.strftime("%s"))
|
|
logging.info("Shutting down Codebase mirror")
|
|
|
|
|
|
# void function that checks the permissions of the files this script needs.
|
|
def check_permissions() -> None:
|
|
# check that the log file can be written
|
|
if config.LOG_FILE:
|
|
try:
|
|
open(config.LOG_FILE, "w")
|
|
except OSError as e:
|
|
sys.stderr.write("Could not open up log for writing:")
|
|
sys.stderr.write(str(e))
|
|
# check that the resume file can be written (this creates if it doesn't exist)
|
|
try:
|
|
open(config.RESUME_FILE, "a+")
|
|
except OSError as e:
|
|
sys.stderr.write(
|
|
"Could not open up the file %s for reading and writing" % (config.RESUME_FILE,)
|
|
)
|
|
sys.stderr.write(str(e))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
assert isinstance(config.RESUME_FILE, str), "RESUME_FILE path not given; refusing to continue"
|
|
check_permissions()
|
|
if config.LOG_FILE:
|
|
logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
|
|
else:
|
|
logging.basicConfig(level=logging.WARNING)
|
|
run_mirror()
|