python-zulip-api/tools/test-bots

#!/usr/bin/env python3

from os.path import dirname, basename
from importlib import import_module

import os
import sys
import argparse
import glob
import unittest
from unittest import TestCase, TestSuite

def parse_args():
    description = """
Script to run test_<bot_name>.py files in the
zulip_bot/zulip_bots/bots/<bot_name> directories.

Running all tests:

./test-bots

Running tests for specific bots:

./test-bots define xkcd

Running all tests excluding certain bots (the
following command would run tests for all bots except
the tests for xkcd and wikipedia bots):

./test-bots --exclude xkcd wikipedia
"""
    parser = argparse.ArgumentParser(description=description)

    parser.add_argument('bots_to_test',
                        metavar='bot',
                        nargs='*',
                        default=[],
                        help='specific bots to test (default is all)')
    parser.add_argument('--coverage',
                        nargs='?',
                        const=True,
                        default=False,
                        help='compute test coverage (--coverage combine to combine with previous reports)')
    parser.add_argument('--exclude',
                        metavar='bot',
                        nargs='*',
                        default=[],
                        help='bot(s) to exclude')
    return parser.parse_args()


def main():
    TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
    os.chdir(os.path.dirname(TOOLS_DIR))
    sys.path.insert(0, TOOLS_DIR)
    bots_dir = os.path.join(TOOLS_DIR, '..', 'zulip_bots/zulip_bots/bots')
    glob_pattern = bots_dir + '/*/test_*.py'
    test_modules = glob.glob(glob_pattern)

    # get only the names of bots that have tests
    available_bots = map(lambda path: basename(dirname(path)), test_modules)

    options = parse_args()

    if options.coverage:
        import coverage
        cov = coverage.Coverage(config_file="tools/.coveragerc")
        if options.coverage == 'combine':
            cov.load()
        cov.start()

    if options.bots_to_test:
        bots_to_test = filter(lambda bot: bot not in options.exclude,
                              options.bots_to_test)
    else:
        bots_to_test = filter(lambda bot: bot not in options.exclude,
                              available_bots)

    # Should add a check here that __init__.py is absent if test_*.py is present?

    # Codecov seems to work only when using loader.discover. It failed to
    # capture line executions for functions like loader.loadTestFromModule
    # or loader.loadTestFromNames.
    top_level = "zulip_bots/zulip_bots/bots/"
    loader = unittest.defaultTestLoader
    test_suites = [loader.discover(top_level + name, top_level_dir=top_level)
                   for name in bots_to_test]

    def filter_tests(tests):
        # type: (Union[TestSuite, TestCase]) -> TestSuite
        filtered_tests = TestSuite()
        for test in tests:
            if isinstance(test, TestCase):
                # Exclude test base class from being tested.
                if test.__class__.__name__ != 'BotTestCase':
                    filtered_tests.addTest(test)
            else:
                filtered_tests.addTest(filter_tests(test))
        return filtered_tests
    test_suites = filter_tests(test_suites)

    suite = unittest.TestSuite(test_suites)
    runner = unittest.TextTestRunner(verbosity=2)
    result = runner.run(suite)
    if result.failures or result.errors:
        sys.exit(1)

    if not result.failures and options.coverage:
        cov.stop()
        cov.data_suffix = False  # Disable suffix so that filename is .coverage
        cov.save()
        cov.html_report()
        print("HTML report saved under directory 'htmlcov'.")

if __name__ == '__main__':
    main()