Replace the custom test runners with pytest.
Signed-off-by: Anders Kaseorg <anders@zulip.com>
This commit is contained in:
Anders Kaseorg
parent
cb00a29311
commit
564fc0e527
9
.github/workflows/zulip-tests.yml
vendored
9
.github/workflows/zulip-tests.yml
vendored
|
|
@ -52,13 +52,14 @@ jobs:
|
||||||
name: Running Test-Suite on Linux
|
name: Running Test-Suite on Linux
|
||||||
run: |
|
run: |
|
||||||
source zulip-api-py3-venv/bin/activate
|
source zulip-api-py3-venv/bin/activate
|
||||||
tools/test-main
|
pytest --cov --cov-config=tools/.coveragerc --cov-report=xml
|
||||||
|
|
||||||
- if: runner.os == 'Windows'
|
- if: runner.os == 'Windows'
|
||||||
name: Running Test-Suite on Windows
|
name: Running Test-Suite on Windows
|
||||||
run: |
|
run: |
|
||||||
zulip-api-py3-venv\Scripts\Activate.ps1
|
zulip-api-py3-venv\Scripts\Activate.ps1
|
||||||
tools\test-main.ps1
|
pytest --cov --cov-config=tools\.coveragerc --cov-report=xml
|
||||||
|
|
||||||
- name: Codecov
|
- uses: codecov/codecov-action@v2
|
||||||
uses: codecov/codecov-action@v1.2.1
|
with:
|
||||||
|
files: coverage.xml
|
||||||
|
|
|
||||||
11
README.md
11
README.md
|
|
@ -59,13 +59,14 @@ and [commit guidelines](https://zulip.readthedocs.io/en/latest/contributing/vers
|
||||||
|
|
||||||
### Running tests
|
### Running tests
|
||||||
|
|
||||||
To run the tests for
|
You can run all the tests with:
|
||||||
|
|
||||||
* *zulip*: run `./tools/test-zulip`
|
`pytest`
|
||||||
|
|
||||||
* *zulip_bots*: run `./tools/test-lib && ./tools/test-bots`
|
or test individual packages with `pytest zulip`, `pytest zulip_bots`,
|
||||||
|
or `pytest zulip_botserver` (see the [pytest
|
||||||
* *zulip_botserver*: run `./tools/test-botserver`
|
documentation](https://docs.pytest.org/en/latest/how-to/usage.html)
|
||||||
|
for more options).
|
||||||
|
|
||||||
To run the linter, type:
|
To run the linter, type:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
crayons
|
crayons
|
||||||
twine
|
twine
|
||||||
coverage>=4.4.1
|
|
||||||
black
|
black
|
||||||
isort
|
isort
|
||||||
flake8
|
flake8
|
||||||
mock
|
mock
|
||||||
pytest
|
pytest
|
||||||
|
pytest-cov
|
||||||
-e ./zulip
|
-e ./zulip
|
||||||
-e ./zulip_bots
|
-e ./zulip_bots
|
||||||
-e ./zulip_botserver
|
-e ./zulip_botserver
|
||||||
|
|
|
||||||
|
|
@ -14,9 +14,3 @@ exclude_lines =
|
||||||
|
|
||||||
[run]
|
[run]
|
||||||
source = zulip, zulip_bots, zulip_botserver
|
source = zulip, zulip_bots, zulip_botserver
|
||||||
omit =
|
|
||||||
# Parts of the test runner infrastructure
|
|
||||||
tools/test-main
|
|
||||||
tools/test-bots
|
|
||||||
tools/test-botserver
|
|
||||||
tools/test-zulip
|
|
||||||
|
|
|
||||||
|
|
@ -1,76 +0,0 @@
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
||||||
os.chdir(os.path.dirname(TOOLS_DIR))
|
|
||||||
|
|
||||||
|
|
||||||
def handle_input_and_run_tests_for_package(package_name, path_list):
|
|
||||||
parser = argparse.ArgumentParser(description=f"Run tests for {package_name}.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--coverage",
|
|
||||||
nargs="?",
|
|
||||||
const=True,
|
|
||||||
default=False,
|
|
||||||
help="compute test coverage (--coverage combine to combine with previous reports)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--pytest", "-p", default=False, action="store_true", help="run tests with pytest"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--verbose",
|
|
||||||
"-v",
|
|
||||||
default=False,
|
|
||||||
action="store_true",
|
|
||||||
help="show verbose output (with pytest)",
|
|
||||||
)
|
|
||||||
options = parser.parse_args()
|
|
||||||
|
|
||||||
test_session_title = f" Running tests for {package_name} "
|
|
||||||
header = test_session_title.center(shutil.get_terminal_size().columns, "#")
|
|
||||||
print(header)
|
|
||||||
|
|
||||||
if options.coverage:
|
|
||||||
import coverage
|
|
||||||
|
|
||||||
cov = coverage.Coverage(config_file="tools/.coveragerc")
|
|
||||||
if options.coverage == "combine":
|
|
||||||
cov.load()
|
|
||||||
cov.start()
|
|
||||||
|
|
||||||
if options.pytest:
|
|
||||||
location_to_run_in = os.path.join(TOOLS_DIR, "..", *path_list)
|
|
||||||
paths_to_test = ["."]
|
|
||||||
pytest_options = [
|
|
||||||
"-s", # show output from tests; this hides the progress bar though
|
|
||||||
"-x", # stop on first test failure
|
|
||||||
"--ff", # runs last failure first
|
|
||||||
]
|
|
||||||
pytest_options += ["-v"] if options.verbose else []
|
|
||||||
os.chdir(location_to_run_in)
|
|
||||||
result = pytest.main(paths_to_test + pytest_options)
|
|
||||||
if result != 0:
|
|
||||||
sys.exit(1)
|
|
||||||
failures = False
|
|
||||||
else:
|
|
||||||
# Codecov seems to work only when using loader.discover. It failed to capture line executions
|
|
||||||
# for functions like loader.loadTestFromModule or loader.loadTestFromNames.
|
|
||||||
test_suites = unittest.defaultTestLoader.discover(os.path.join(*path_list))
|
|
||||||
suite = unittest.TestSuite(test_suites)
|
|
||||||
runner = unittest.TextTestRunner(verbosity=2)
|
|
||||||
result = runner.run(suite)
|
|
||||||
failures = result.failures
|
|
||||||
if result.failures or result.errors:
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if not failures and options.coverage:
|
|
||||||
cov.stop()
|
|
||||||
cov.data_suffix = False # Disable suffix so that filename is .coverage
|
|
||||||
cov.save()
|
|
||||||
cov.html_report()
|
|
||||||
print("HTML report saved in directory 'htmlcov'.")
|
|
||||||
144
tools/test-bots
144
tools/test-bots
|
|
@ -1,144 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import unittest
|
|
||||||
from os.path import basename, dirname
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
|
||||||
description = """
|
|
||||||
Script to run test_<bot_name>.py files in the
|
|
||||||
zulip_bot/zulip_bots/bots/<bot_name> directories.
|
|
||||||
|
|
||||||
Running all tests:
|
|
||||||
|
|
||||||
./test-bots
|
|
||||||
|
|
||||||
Running tests for specific bots:
|
|
||||||
|
|
||||||
./test-bots define xkcd
|
|
||||||
|
|
||||||
Running all tests excluding certain bots (the
|
|
||||||
following command would run tests for all bots except
|
|
||||||
the tests for xkcd and wikipedia bots):
|
|
||||||
|
|
||||||
./test-bots --exclude xkcd wikipedia
|
|
||||||
"""
|
|
||||||
parser = argparse.ArgumentParser(description=description)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"bots_to_test",
|
|
||||||
metavar="bot",
|
|
||||||
nargs="*",
|
|
||||||
default=[],
|
|
||||||
help="specific bots to test (default is all)",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--coverage",
|
|
||||||
nargs="?",
|
|
||||||
const=True,
|
|
||||||
default=False,
|
|
||||||
help="compute test coverage (--coverage combine to combine with previous reports)",
|
|
||||||
)
|
|
||||||
parser.add_argument("--exclude", metavar="bot", nargs="*", default=[], help="bot(s) to exclude")
|
|
||||||
parser.add_argument(
|
|
||||||
"--error-on-no-init",
|
|
||||||
default=False,
|
|
||||||
action="store_true",
|
|
||||||
help="whether to exit if a bot has tests which won't run due to no __init__.py",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--pytest", "-p", default=False, action="store_true", help="run tests with pytest"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--verbose",
|
|
||||||
"-v",
|
|
||||||
default=False,
|
|
||||||
action="store_true",
|
|
||||||
help="show verbose output (with pytest)",
|
|
||||||
)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
os.chdir(os.path.dirname(TOOLS_DIR))
|
|
||||||
sys.path.insert(0, TOOLS_DIR)
|
|
||||||
bots_dir = os.path.join(TOOLS_DIR, "..", "zulip_bots/zulip_bots/bots")
|
|
||||||
glob_pattern = bots_dir + "/*/test_*.py"
|
|
||||||
test_modules = glob.glob(glob_pattern)
|
|
||||||
|
|
||||||
# get only the names of bots that have tests
|
|
||||||
available_bots = map(lambda path: basename(dirname(path)), test_modules)
|
|
||||||
|
|
||||||
options = parse_args()
|
|
||||||
|
|
||||||
if options.coverage:
|
|
||||||
import coverage
|
|
||||||
|
|
||||||
cov = coverage.Coverage(config_file="tools/.coveragerc")
|
|
||||||
if options.coverage == "combine":
|
|
||||||
cov.load()
|
|
||||||
cov.start()
|
|
||||||
|
|
||||||
if options.bots_to_test:
|
|
||||||
specified_bots = options.bots_to_test
|
|
||||||
else:
|
|
||||||
specified_bots = available_bots
|
|
||||||
|
|
||||||
# Use of a set ensures we don't end up with duplicate tests with unittest
|
|
||||||
# (from globbing multiple test_*.py files, or multiple on the command line)
|
|
||||||
bots_to_test = {bot for bot in specified_bots if bot not in options.exclude}
|
|
||||||
|
|
||||||
if options.pytest:
|
|
||||||
pytest_options = [
|
|
||||||
"-s", # show output from tests; this hides the progress bar though
|
|
||||||
"-x", # stop on first test failure
|
|
||||||
"--ff", # runs last failure first
|
|
||||||
]
|
|
||||||
pytest_options += ["-v"] if options.verbose else []
|
|
||||||
os.chdir(bots_dir)
|
|
||||||
result = pytest.main(sorted(bots_to_test) + pytest_options)
|
|
||||||
if result != 0:
|
|
||||||
sys.exit(1)
|
|
||||||
failures = False
|
|
||||||
else:
|
|
||||||
# Codecov seems to work only when using loader.discover. It failed to
|
|
||||||
# capture line executions for functions like loader.loadTestFromModule
|
|
||||||
# or loader.loadTestFromNames.
|
|
||||||
top_level = "zulip_bots/zulip_bots/bots/"
|
|
||||||
loader = unittest.defaultTestLoader
|
|
||||||
test_suites = []
|
|
||||||
for name in bots_to_test:
|
|
||||||
try:
|
|
||||||
test_suites.append(loader.discover(top_level + name, top_level_dir=top_level))
|
|
||||||
except ImportError as exception:
|
|
||||||
print(exception)
|
|
||||||
print(
|
|
||||||
"This likely indicates that you need a '__init__.py' file in your bot directory."
|
|
||||||
)
|
|
||||||
if options.error_on_no_init:
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
suite = unittest.TestSuite(test_suites)
|
|
||||||
runner = unittest.TextTestRunner(verbosity=2)
|
|
||||||
result = runner.run(suite)
|
|
||||||
failures = result.failures
|
|
||||||
if failures or result.errors:
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if not failures and options.coverage:
|
|
||||||
cov.stop()
|
|
||||||
cov.data_suffix = False # Disable suffix so that filename is .coverage
|
|
||||||
cov.save()
|
|
||||||
cov.html_report()
|
|
||||||
print("HTML report saved under directory 'htmlcov'.")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from server_lib.test_handler import handle_input_and_run_tests_for_package
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
handle_input_and_run_tests_for_package("Botserver", ["zulip_botserver"])
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from server_lib.test_handler import handle_input_and_run_tests_for_package
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
handle_input_and_run_tests_for_package("Bot library", ["zulip_bots", "zulip_bots", "tests"])
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -ev
|
|
||||||
|
|
||||||
tools/test-bots "$@" --coverage
|
|
||||||
tools/test-botserver "$@" --coverage combine
|
|
||||||
tools/test-zulip "$@" --coverage combine
|
|
||||||
tools/test-lib "$@" --coverage combine
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = "Stop"
|
|
||||||
|
|
||||||
python tools\test-bots "$args" --coverage
|
|
||||||
python tools\test-botserver "$args" --coverage combine
|
|
||||||
python tools\test-zulip "$args" --coverage combine
|
|
||||||
python tools\test-lib "$args" --coverage combine
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from server_lib.test_handler import handle_input_and_run_tests_for_package
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
handle_input_and_run_tests_for_package("API", ["zulip"])
|
|
||||||
Loading…
Reference in a new issue