python-zulip-api/tools/test-bots

#!/usr/bin/env python3

from os.path import dirname, basename
from importlib import import_module

import os
import sys
import argparse
import glob
import unittest
from unittest import TestCase, TestSuite
import pytest

def parse_args():
    description = """
Script to run test_<bot_name>.py files in the
zulip_bot/zulip_bots/bots/<bot_name> directories.

Running all tests:

./test-bots

Running tests for specific bots:

./test-bots define xkcd

Running all tests excluding certain bots (the
following command would run tests for all bots except
the tests for xkcd and wikipedia bots):

./test-bots --exclude xkcd wikipedia
"""
    parser = argparse.ArgumentParser(description=description)

    parser.add_argument('bots_to_test',
                        metavar='bot',
                        nargs='*',
                        default=[],
                        help='specific bots to test (default is all)')
    parser.add_argument('--coverage',
                        nargs='?',
                        const=True,
                        default=False,
                        help='compute test coverage (--coverage combine to combine with previous reports)')
    parser.add_argument('--exclude',
                        metavar='bot',
                        nargs='*',
                        default=[],
                        help='bot(s) to exclude')
    parser.add_argument('--error-on-no-init',
                        default=False,
                        action="store_true",
                        help="whether to exit if a bot has tests which won't run due to no __init__.py")
    parser.add_argument('--pytest', '-p',
                        default=False,
                        action='store_true',
                        help="run tests with pytest")
    parser.add_argument('--verbose', '-v',
                        default=False,
                        action='store_true',
                        help='show verbose output (with pytest)')
    return parser.parse_args()


def main():
    TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
    os.chdir(os.path.dirname(TOOLS_DIR))
    sys.path.insert(0, TOOLS_DIR)
    bots_dir = os.path.join(TOOLS_DIR, '..', 'zulip_bots/zulip_bots/bots')
    glob_pattern = bots_dir + '/*/test_*.py'
    test_modules = glob.glob(glob_pattern)

    # get only the names of bots that have tests
    available_bots = map(lambda path: basename(dirname(path)), test_modules)

    options = parse_args()

    if options.coverage:
        import coverage
        cov = coverage.Coverage(config_file="tools/.coveragerc")
        if options.coverage == 'combine':
            cov.load()
        cov.start()

    if options.bots_to_test:
        specified_bots = options.bots_to_test
    else:
        specified_bots = available_bots

    bots_to_test = filter(lambda bot: bot not in options.exclude, specified_bots)

    if options.pytest:
        excluded_bots = ['merels']
        pytest_bots_to_test = sorted([bot for bot in bots_to_test if bot not in excluded_bots])
        pytest_options = [
            '-s',    # show output from tests; this hides the progress bar though
            '-x',    # stop on first test failure
            '--ff',  # runs last failure first
        ]
        pytest_options += (['-v'] if options.verbose else [])
        os.chdir(bots_dir)
        result = pytest.main(pytest_bots_to_test + pytest_options)
        if result != 0:
            sys.exit(1)
        failures = False
    else:
        # Codecov seems to work only when using loader.discover. It failed to
        # capture line executions for functions like loader.loadTestFromModule
        # or loader.loadTestFromNames.
        top_level = "zulip_bots/zulip_bots/bots/"
        loader = unittest.defaultTestLoader
        test_suites = []
        for name in bots_to_test:
            try:
                test_suites.append(loader.discover(top_level + name, top_level_dir=top_level))
            except ImportError as exception:
                print(exception)
                print("This likely indicates that you need a '__init__.py' file in your bot directory.")
                if options.error_on_no_init:
                    sys.exit(1)

        suite = unittest.TestSuite(test_suites)
        runner = unittest.TextTestRunner(verbosity=2)
        result = runner.run(suite)
        failures = result.failures
        if failures or result.errors:
            sys.exit(1)

    if not failures and options.coverage:
        cov.stop()
        cov.data_suffix = False  # Disable suffix so that filename is .coverage
        cov.save()
        cov.html_report()
        print("HTML report saved under directory 'htmlcov'.")

if __name__ == '__main__':
    main()