#!/usr/bin/env python3 import argparse import glob import os import sys import unittest from os.path import basename, dirname import pytest def parse_args(): description = """ Script to run test_.py files in the zulip_bot/zulip_bots/bots/ directories. Running all tests: ./test-bots Running tests for specific bots: ./test-bots define xkcd Running all tests excluding certain bots (the following command would run tests for all bots except the tests for xkcd and wikipedia bots): ./test-bots --exclude xkcd wikipedia """ parser = argparse.ArgumentParser(description=description) parser.add_argument( "bots_to_test", metavar="bot", nargs="*", default=[], help="specific bots to test (default is all)", ) parser.add_argument( "--coverage", nargs="?", const=True, default=False, help="compute test coverage (--coverage combine to combine with previous reports)", ) parser.add_argument("--exclude", metavar="bot", nargs="*", default=[], help="bot(s) to exclude") parser.add_argument( "--error-on-no-init", default=False, action="store_true", help="whether to exit if a bot has tests which won't run due to no __init__.py", ) parser.add_argument( "--pytest", "-p", default=False, action="store_true", help="run tests with pytest" ) parser.add_argument( "--verbose", "-v", default=False, action="store_true", help="show verbose output (with pytest)", ) return parser.parse_args() def main(): TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.dirname(TOOLS_DIR)) sys.path.insert(0, TOOLS_DIR) bots_dir = os.path.join(TOOLS_DIR, "..", "zulip_bots/zulip_bots/bots") glob_pattern = bots_dir + "/*/test_*.py" test_modules = glob.glob(glob_pattern) # get only the names of bots that have tests available_bots = map(lambda path: basename(dirname(path)), test_modules) options = parse_args() if options.coverage: import coverage cov = coverage.Coverage(config_file="tools/.coveragerc") if options.coverage == "combine": cov.load() cov.start() if options.bots_to_test: specified_bots = options.bots_to_test else: specified_bots = available_bots # Use of a set ensures we don't end up with duplicate tests with unittest # (from globbing multiple test_*.py files, or multiple on the command line) bots_to_test = {bot for bot in specified_bots if bot not in options.exclude} if options.pytest: pytest_options = [ "-s", # show output from tests; this hides the progress bar though "-x", # stop on first test failure "--ff", # runs last failure first ] pytest_options += ["-v"] if options.verbose else [] os.chdir(bots_dir) result = pytest.main(sorted(bots_to_test) + pytest_options) if result != 0: sys.exit(1) failures = False else: # Codecov seems to work only when using loader.discover. It failed to # capture line executions for functions like loader.loadTestFromModule # or loader.loadTestFromNames. top_level = "zulip_bots/zulip_bots/bots/" loader = unittest.defaultTestLoader test_suites = [] for name in bots_to_test: try: test_suites.append(loader.discover(top_level + name, top_level_dir=top_level)) except ImportError as exception: print(exception) print( "This likely indicates that you need a '__init__.py' file in your bot directory." ) if options.error_on_no_init: sys.exit(1) suite = unittest.TestSuite(test_suites) runner = unittest.TextTestRunner(verbosity=2) result = runner.run(suite) failures = result.failures if failures or result.errors: sys.exit(1) if not failures and options.coverage: cov.stop() cov.data_suffix = False # Disable suffix so that filename is .coverage cov.save() cov.html_report() print("HTML report saved under directory 'htmlcov'.") if __name__ == "__main__": main()