python-zulip-api/zulip/integrations/zephyr/sync-public-streams

#!/usr/bin/env python3
import sys
import os
import logging
import argparse
import time
import json
import subprocess
import unicodedata

sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'api'))
import zulip

from typing import Set, Optional

def fetch_public_streams() -> Optional[Set[bytes]]:
    public_streams = set()

    try:
        res = zulip_client.get_streams(include_all_active=True)
        if res.get("result") == "success":
            streams = res["streams"]
        else:
            logging.error("Error getting public streams:\n%s" % (res,))
            return None
    except Exception:
        logging.exception("Error getting public streams:")
        return None

    for stream in streams:
        stream_name = stream["name"]
        # Zephyr class names are canonicalized by first applying NFKC
        # normalization and then lower-casing server-side
        canonical_cls = unicodedata.normalize("NFKC", stream_name).lower().encode("utf-8")
        if canonical_cls in [b'security', b'login', b'network', b'ops', b'user_locate',
                             b'mit', b'moof', b'wsmonitor', b'wg_ctl', b'winlogger',
                             b'hm_ctl', b'hm_stat', b'zephyr_admin', b'zephyr_ctl']:
            # These zephyr classes cannot be subscribed to by us, due
            # to MIT's Zephyr access control settings
            continue

        public_streams.add(canonical_cls)

    return public_streams

if __name__ == "__main__":
    log_file = "/home/zulip/sync_public_streams.log"
    logger = logging.getLogger(__name__)
    log_format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=log_format)
    formatter = logging.Formatter(log_format)
    logger.setLevel(logging.DEBUG)
    file_handler = logging.FileHandler(log_file)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    parser = zulip.add_default_arguments(argparse.ArgumentParser())
    options = parser.parse_args()

    zulip_client = zulip.Client(client="ZulipSyncPublicStreamsBot/0.1")

    while True:
        # Sync every 5 minutes because this makes a very large network
        # request, due to Zephyr users who subscribe to 10K+ class
        # names generated by a script.
        #
        # This delay means we won't subscribe to new Zephyr classes
        # until 5 minutes after they are created in Zulip; this isn't
        # great but an acceptable tradeoff.
        time.sleep(300)
        public_streams = fetch_public_streams()
        if public_streams is None:
            continue

        f = open("/home/zulip/public_streams.tmp", "w")
        f.write(json.dumps(list(public_streams)) + "\n")
        f.close()

        subprocess.call(["mv", "/home/zulip/public_streams.tmp", "/home/zulip/public_streams"])