tools: Add mypy runner.

Unless otherwise specified, `tools/run-mypy` will right now only
check annotations in core files of the `zulip` package.

.gitignore

@@ -43,3 +43,6 @@ zuliprc
 .zuliprc
 flaskbotrc
 .flaskbotrc
+
+# mypy
+.mypy_cache

requirements.txt

@@ -1,4 +1,5 @@
 coverage>=4.4.1
+mypy==0.521
 pycodestyle==2.3.1
 -e ./zulip
 -e ./zulip_bots

tools/lister.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+from __future__ import print_function
+from __future__ import absolute_import
+
+import os
+from os.path import abspath
+import sys
+import subprocess
+import re
+from collections import defaultdict
+import argparse
+from six.moves import filter
+
+from typing import Union, List, Dict
+
+def get_ftype(fpath, use_shebang):
+    # type: (str, bool) -> str
+    ext = os.path.splitext(fpath)[1]
+    if ext:
+        return ext[1:]
+    elif use_shebang:
+        # opening a file may throw an OSError
+        with open(fpath) as f:
+            first_line = f.readline()
+            if re.search(r'^#!.*\bpython', first_line):
+                return 'py'
+            elif re.search(r'^#!.*sh', first_line):
+                return 'sh'
+            elif re.search(r'^#!.*\bperl', first_line):
+                return 'pl'
+            elif re.search(r'^#!.*\bnode', first_line):
+                return 'js'
+            elif re.search(r'^#!.*\bruby', first_line):
+                return 'rb'
+            elif re.search(r'^#!', first_line):
+                print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr)
+                return ''
+            else:
+                return ''
+    else:
+        return ''
+
+def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False,
+               exclude=[], group_by_ftype=False, extless_only=False):
+    # type: (List[str], List[str], bool, bool, List[str], bool, bool) -> Union[Dict[str, List[str]], List[str]]
+    """
+    List files tracked by git.
+
+    Returns a list of files which are either in targets or in directories in targets.
+    If targets is [], list of all tracked files in current directory is returned.
+
+    Other arguments:
+    ftypes - List of file types on which to filter the search.
+        If ftypes is [], all files are included.
+    use_shebang - Determine file type of extensionless files from their shebang.
+    modified_only - Only include files which have been modified.
+    exclude - List of paths to be excluded, relative to repository root.
+    group_by_ftype - If True, returns a dict of lists keyed by file type.
+        If False, returns a flat list of files.
+    extless_only - Only include extensionless files in output.
+    """
+    ftypes = [x.strip('.') for x in ftypes]
+    ftypes_set = set(ftypes)
+
+    # Really this is all bytes -- it's a file path -- but we get paths in
+    # sys.argv as str, so that battle is already lost.  Settle for hoping
+    # everything is UTF-8.
+    repository_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip().decode('utf-8')
+    exclude_abspaths = [os.path.normpath(os.path.join(repository_root, fpath)) for fpath in exclude]
+
+    cmdline = ['git', 'ls-files'] + targets
+    if modified_only:
+        cmdline.append('-m')
+
+    files_gen = (x.strip() for x in subprocess.check_output(cmdline, universal_newlines=True).split('\n'))
+    # throw away empty lines and non-files (like symlinks)
+    files = list(filter(os.path.isfile, files_gen))
+
+    result_dict = defaultdict(list)  # type: Dict[str, List[str]]
+    result_list = []  # type: List[str]
+
+    for fpath in files:
+        # this will take a long time if exclude is very large
+        ext = os.path.splitext(fpath)[1]
+        if extless_only and ext:
+            continue
+        absfpath = abspath(fpath)
+        if any(absfpath == expath or absfpath.startswith(expath + '/')
+               for expath in exclude_abspaths):
+            continue
+
+        if ftypes or group_by_ftype:
+            try:
+                filetype = get_ftype(fpath, use_shebang)
+            except (OSError, UnicodeDecodeError) as e:
+                etype = e.__class__.__name__
+                print('Error: %s while determining type of file "%s":' % (etype, fpath), file=sys.stderr)
+                print(e, file=sys.stderr)
+                filetype = ''
+            if ftypes and filetype not in ftypes_set:
+                continue
+
+        if group_by_ftype:
+            result_dict[filetype].append(fpath)
+        else:
+            result_list.append(fpath)
+
+    if group_by_ftype:
+        return result_dict
+    else:
+        return result_list
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="List files tracked by git and optionally filter by type")
+    parser.add_argument('targets', nargs='*', default=[],
+                        help='''files and directories to include in the result.
+                        If this is not specified, the current directory is used''')
+    parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files')
+    parser.add_argument('-f', '--ftypes', nargs='+', default=[],
+                        help="list of file types to filter on. All files are included if this option is absent")
+    parser.add_argument('--ext-only', dest='extonly', action='store_true', default=False,
+                        help='only use extension to determine file type')
+    parser.add_argument('--exclude', nargs='+', default=[],
+                        help='list of files and directories to exclude from results, relative to repo root')
+    parser.add_argument('--extless-only', dest='extless_only', action='store_true', default=False,
+                        help='only include extensionless files in output')
+    args = parser.parse_args()
+    listing = list_files(targets=args.targets, ftypes=args.ftypes, use_shebang=not args.extonly,
+                         modified_only=args.modified, exclude=args.exclude, extless_only=args.extless_only)
+    for l in listing:
+        print(l)

tools/run-mypy

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+import argparse
+import subprocess
+
+import lister
+from typing import cast, Dict, List
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+os.chdir(os.path.dirname(TOOLS_DIR))
+
+sys.path.append(os.path.dirname(TOOLS_DIR))
+
+exclude = """
+""".split()
+
+default_targets = ['zulip/zulip',
+                   'zulip/setup.py']
+
+parser = argparse.ArgumentParser(description="Run mypy on files tracked by git.")
+parser.add_argument('targets', nargs='*', default=default_targets,
+                    help="""files and directories to include in the result.
+                    If this is not specified, the current directory is used""")
+parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files')
+parser.add_argument('-a', '--all', dest='all', action='store_true', default=False,
+                    help="""run mypy on all python files, ignoring the exclude list.
+                    This is useful if you have to find out which files fail mypy check.""")
+parser.add_argument('--no-disallow-untyped-defs', dest='disallow_untyped_defs', action='store_false', default=True,
+                    help="""Don't throw errors when functions are not annotated""")
+parser.add_argument('--scripts-only', dest='scripts_only', action='store_true', default=False,
+                    help="""Only type check extensionless python scripts""")
+parser.add_argument('--strict-optional', dest='strict_optional', action='store_true', default=False,
+                    help="""Use the --strict-optional flag with mypy""")
+parser.add_argument('--warn-unused-ignores', dest='warn_unused_ignores', action='store_true', default=False,
+                    help="""Use the --warn-unused-ignores flag with mypy""")
+parser.add_argument('--no-ignore-missing-imports', dest='ignore_missing_imports', action='store_false', default=True,
+                    help="""Don't use the --ignore-missing-imports flag with mypy""")
+parser.add_argument('--quick', action='store_true', default=False,
+                    help="""Use the --quick flag with mypy""")
+args = parser.parse_args()
+
+if args.all:
+    exclude = []
+
+# find all non-excluded files in current directory
+files_dict = cast(Dict[str, List[str]],
+                  lister.list_files(targets=args.targets, ftypes=['py', 'pyi'],
+                                    use_shebang=True, modified_only=args.modified,
+                                    exclude = exclude + ['stubs'], group_by_ftype=True,
+                                    extless_only=args.scripts_only))
+pyi_files = set(files_dict['pyi'])
+python_files = [fpath for fpath in files_dict['py']
+                if not fpath.endswith('.py') or fpath + 'i' not in pyi_files]
+
+mypy_command = "mypy"
+
+extra_args = ["--check-untyped-defs",
+              "--follow-imports=silent",
+              "--scripts-are-modules",
+              "-i"]
+if args.disallow_untyped_defs:
+    extra_args.append("--disallow-untyped-defs")
+if args.warn_unused_ignores:
+    extra_args.append("--warn-unused-ignores")
+if args.strict_optional:
+    extra_args.append("--strict-optional")
+if args.ignore_missing_imports:
+    extra_args.append("--ignore-missing-imports")
+if args.quick:
+    extra_args.append("--quick")
+
+# run mypy
+if python_files:
+    rc = subprocess.call([mypy_command] + extra_args + python_files)
+    sys.exit(rc)
+else:
+    print("There are no files to run mypy on.")