#!/usr/bin/env python3 import sys import os import logging import argparse import time import json import subprocess import unicodedata sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'api')) import zulip from typing import Set, Optional def fetch_public_streams() -> Optional[Set[str]]: public_streams = set() try: res = zulip_client.get_streams(include_all_active=True) if res.get("result") == "success": streams = res["streams"] else: logging.error("Error getting public streams:\n%s" % (res,)) return None except Exception: logging.exception("Error getting public streams:") return None for stream in streams: stream_name = stream["name"] # Zephyr class names are canonicalized by first applying NFKC # normalization and then lower-casing server-side canonical_cls = unicodedata.normalize("NFKC", stream_name).lower() if canonical_cls in ['security', 'login', 'network', 'ops', 'user_locate', 'mit', 'moof', 'wsmonitor', 'wg_ctl', 'winlogger', 'hm_ctl', 'hm_stat', 'zephyr_admin', 'zephyr_ctl']: # These zephyr classes cannot be subscribed to by us, due # to MIT's Zephyr access control settings continue public_streams.add(canonical_cls) return public_streams if __name__ == "__main__": log_file = "/home/zulip/sync_public_streams.log" logger = logging.getLogger(__name__) log_format = "%(asctime)s: %(message)s" logging.basicConfig(format=log_format) formatter = logging.Formatter(log_format) logger.setLevel(logging.DEBUG) file_handler = logging.FileHandler(log_file) file_handler.setFormatter(formatter) logger.addHandler(file_handler) parser = zulip.add_default_arguments(argparse.ArgumentParser()) options = parser.parse_args() zulip_client = zulip.Client(client="ZulipSyncPublicStreamsBot/0.1") while True: # Sync every 5 minutes because this makes a very large network # request, due to Zephyr users who subscribe to 10K+ class # names generated by a script. # # This delay means we won't subscribe to new Zephyr classes # until 5 minutes after they are created in Zulip; this isn't # great but an acceptable tradeoff. time.sleep(300) public_streams = fetch_public_streams() if public_streams is None: continue f = open("/home/zulip/public_streams.tmp", "w") f.write(json.dumps(list(public_streams)) + "\n") f.close() subprocess.call(["mv", "/home/zulip/public_streams.tmp", "/home/zulip/public_streams"])