testing: Add code test coverage reporting.

Fixes #38.

.codecov.yml

@@ -0,0 +1,10 @@
+comment: off
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0.03
+        base: auto
+    patch: off

.travis.yml

@@ -10,9 +10,17 @@ matrix:
   - python: "3.6"
     env: TEST_SUITE=lint
 install:
+    # Install codecov, the library for the code coverage reporting tool we use
+  - pip install codecov
+
+    # Install pycodestyle, the library for checking pep8 code conformity.
   - pip install pycodestyle==2.3.1
+
+    # Install all API packages
   - pip install ./zulip
   - pip install ./zulip_bots
   - pip install ./zulip_botserver
 script:
   - tools/$TEST_SUITE
+after_success:
+  - codecov

tools/.coveragerc

@@ -0,0 +1,24 @@
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Re-enable the standard coverage pragma
+    nocoverage
+    # Don't complain if non-runnable code isn't run:
+    if False:
+    # Don't require coverage for base class NotImplementedErrors
+    raise NotImplementedError()
+    # Don't require coverage for test suite AssertionError -- they're usually for clarity
+    raise AssertionError
+    # Don't require coverage for Python 2/3 variations; it's impossible to cover on both
+    if six.PY3:
+    if six.PY2:
+    # Don't require coverage for __unicode__ statements just used for printing
+    def __unicode__[(]self[)]:
+
+[run]
+source = zulip_bots, zulip_botserver
+omit =
+    # Parts of the test runner infrastructure
+    tools/test-main
+    tools/test-bots
+    tools/test-botserver

tools/test-bots

@@ -2,31 +2,32 @@
 
 from os.path import dirname, basename
 from importlib import import_module
-import argparse
+
 import os
 import sys
+import argparse
 import glob
 import unittest
-import logging
-
 
 def load_tests_from_modules(names, template):
     loader = unittest.defaultTestLoader
     test_suites = []
-
     for name in names:
         module = import_module(template.format(name=name))
         test_suites.append(loader.loadTestsFromModule(module))
 
     return test_suites
 
+def load_all_tests():
+    loader = unittest.defaultTestLoader
+    return loader.discover('zulip_bots')
 
 def parse_args(available_bots):
     description = """
 Script to run test_<bot_name>.py files in the
 zulip_bot/zulip_bots/bots/<bot_name> directories.
 
-Running tests for all bots:
+Running all tests:
 
 ./test-bots
 
@@ -34,7 +35,7 @@ Running tests for specific bots:
 
 ./test-bots define xkcd
 
-Running tests for all bots excluding certain bots (the
+Running all tests excluding certain bots (the
 following command would run tests for all bots except
 the tests for xkcd and wikipedia bots):
 
@@ -45,9 +46,13 @@ the tests for xkcd and wikipedia bots):
     parser.add_argument('bots_to_test',
                         metavar='bot',
                         nargs='*',
-                        default=available_bots,
+                        default=[],
                         help='specific bots to test (default is all)')
-
+    parser.add_argument('--coverage',
+                        nargs='?',
+                        const=True,
+                        default=False,
+                        help='compute test coverage (--coverage combine to combine with previous reports)')
     parser.add_argument('--exclude',
                         metavar='bot',
                         nargs='*',
@@ -57,22 +62,32 @@ the tests for xkcd and wikipedia bots):
 
 
 def main():
-    current_dir = os.path.dirname(os.path.abspath(__file__))
-    bots_dir = os.path.join(current_dir, '..', 'zulip_bots/zulip_bots/bots')
+    TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+    os.chdir(os.path.dirname(TOOLS_DIR))
+    sys.path.insert(0, TOOLS_DIR)
+    bots_dir = os.path.join(TOOLS_DIR, '..', 'zulip_bots/zulip_bots/bots')
     glob_pattern = bots_dir + '/*/test_*.py'
     test_modules = glob.glob(glob_pattern)
 
     # get only the names of bots that have tests
     available_bots = map(lambda path: basename(dirname(path)), test_modules)
-
     options = parse_args(available_bots)
 
-    bots_to_test = filter(lambda bot: bot not in options.exclude,
-                          options.bots_to_test)
-    test_suites = load_tests_from_modules(
-        bots_to_test,
-        template='zulip_bots.bots.{name}.test_{name}'
-    )
+    if options.coverage:
+        import coverage
+        cov = coverage.Coverage(config_file="tools/.coveragerc")
+        if options.coverage == 'combine':
+            cov.load()
+        cov.start()
+
+    if options.bots_to_test:
+        bots_to_test = filter(lambda bot: bot not in options.exclude,
+                              options.bots_to_test)
+        test_suites = load_tests_from_modules(
+            bots_to_test,
+            template='zulip_bots.bots.{name}.test_{name}')
+    else:
+        test_suites = load_all_tests()
 
     suite = unittest.TestSuite(test_suites)
     runner = unittest.TextTestRunner(verbosity=2)
@@ -80,6 +95,12 @@ def main():
     if result.failures or result.errors:
         sys.exit(1)
 
+    if not result.failures and options.coverage:
+        cov.stop()
+        cov.data_suffix = False  # Disable suffix so that filename is .coverage
+        cov.save()
+        cov.html_report()
+        print("HTML report saved under directory 'htmlcov'.")
 
 if __name__ == '__main__':
     main()

tools/test-botserver

@@ -13,13 +13,17 @@ def main():
 
     parser = argparse.ArgumentParser(description="Run tests for the zulip_botserver package.")
     parser.add_argument('--coverage',
-                        action='store_true',
-                        help='compute test coverage')
+                        nargs='?',
+                        const=True,
+                        default=False,
+                        help='compute test coverage (--coverage combine to combine with previous reports)')
     options = parser.parse_args()
 
     if options.coverage:
         import coverage
         cov = coverage.Coverage(config_file="tools/.coveragerc")
+        if options.coverage == 'combine':
+            cov.load()
         cov.start()
 
     test_suites = unittest.defaultTestLoader.discover('zulip_botserver')
@@ -31,8 +35,6 @@ def main():
 
     if not result.failures and options.coverage:
         cov.stop()
-        cov.save()
-        cov.combine()
         cov.data_suffix = False  # Disable suffix so that filename is .coverage
         cov.save()
         cov.html_report()

tools/test-main

@@ -2,8 +2,5 @@
 
 set -ev
 
-CURRENT_DIR=$(dirname $(dirname "$0"))
-cd "$CURRENT_DIR"
-
-python -m unittest discover -v zulip_botserver
-tools/test-bots
+tools/test-bots --coverage combine
+tools/test-botserver --coverage combine