From 035f0c32680ca7d5bb55044e841c2fac4cd6701f Mon Sep 17 00:00:00 2001 From: derAnfaenger Date: Fri, 15 Sep 2017 13:24:20 +0200 Subject: [PATCH] tools: Add mypy runner. Unless otherwise specified, `tools/run-mypy` will right now only check annotations in core files of the `zulip` package. --- .gitignore | 3 ++ requirements.txt | 1 + tools/lister.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++ tools/run-mypy | 82 +++++++++++++++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 tools/lister.py create mode 100755 tools/run-mypy diff --git a/.gitignore b/.gitignore index 1a8569b..ec49d56 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ zuliprc .zuliprc flaskbotrc .flaskbotrc + +# mypy +.mypy_cache diff --git a/requirements.txt b/requirements.txt index c036c99..f102785 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ coverage>=4.4.1 +mypy==0.521 pycodestyle==2.3.1 -e ./zulip -e ./zulip_bots diff --git a/tools/lister.py b/tools/lister.py new file mode 100644 index 0000000..6e8e547 --- /dev/null +++ b/tools/lister.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +from __future__ import print_function +from __future__ import absolute_import + +import os +from os.path import abspath +import sys +import subprocess +import re +from collections import defaultdict +import argparse +from six.moves import filter + +from typing import Union, List, Dict + +def get_ftype(fpath, use_shebang): + # type: (str, bool) -> str + ext = os.path.splitext(fpath)[1] + if ext: + return ext[1:] + elif use_shebang: + # opening a file may throw an OSError + with open(fpath) as f: + first_line = f.readline() + if re.search(r'^#!.*\bpython', first_line): + return 'py' + elif re.search(r'^#!.*sh', first_line): + return 'sh' + elif re.search(r'^#!.*\bperl', first_line): + return 'pl' + elif re.search(r'^#!.*\bnode', first_line): + return 'js' + elif re.search(r'^#!.*\bruby', first_line): + return 'rb' + elif re.search(r'^#!', first_line): + print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr) + return '' + else: + return '' + else: + return '' + +def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False, + exclude=[], group_by_ftype=False, extless_only=False): + # type: (List[str], List[str], bool, bool, List[str], bool, bool) -> Union[Dict[str, List[str]], List[str]] + """ + List files tracked by git. + + Returns a list of files which are either in targets or in directories in targets. + If targets is [], list of all tracked files in current directory is returned. + + Other arguments: + ftypes - List of file types on which to filter the search. + If ftypes is [], all files are included. + use_shebang - Determine file type of extensionless files from their shebang. + modified_only - Only include files which have been modified. + exclude - List of paths to be excluded, relative to repository root. + group_by_ftype - If True, returns a dict of lists keyed by file type. + If False, returns a flat list of files. + extless_only - Only include extensionless files in output. + """ + ftypes = [x.strip('.') for x in ftypes] + ftypes_set = set(ftypes) + + # Really this is all bytes -- it's a file path -- but we get paths in + # sys.argv as str, so that battle is already lost. Settle for hoping + # everything is UTF-8. + repository_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip().decode('utf-8') + exclude_abspaths = [os.path.normpath(os.path.join(repository_root, fpath)) for fpath in exclude] + + cmdline = ['git', 'ls-files'] + targets + if modified_only: + cmdline.append('-m') + + files_gen = (x.strip() for x in subprocess.check_output(cmdline, universal_newlines=True).split('\n')) + # throw away empty lines and non-files (like symlinks) + files = list(filter(os.path.isfile, files_gen)) + + result_dict = defaultdict(list) # type: Dict[str, List[str]] + result_list = [] # type: List[str] + + for fpath in files: + # this will take a long time if exclude is very large + ext = os.path.splitext(fpath)[1] + if extless_only and ext: + continue + absfpath = abspath(fpath) + if any(absfpath == expath or absfpath.startswith(expath + '/') + for expath in exclude_abspaths): + continue + + if ftypes or group_by_ftype: + try: + filetype = get_ftype(fpath, use_shebang) + except (OSError, UnicodeDecodeError) as e: + etype = e.__class__.__name__ + print('Error: %s while determining type of file "%s":' % (etype, fpath), file=sys.stderr) + print(e, file=sys.stderr) + filetype = '' + if ftypes and filetype not in ftypes_set: + continue + + if group_by_ftype: + result_dict[filetype].append(fpath) + else: + result_list.append(fpath) + + if group_by_ftype: + return result_dict + else: + return result_list + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="List files tracked by git and optionally filter by type") + parser.add_argument('targets', nargs='*', default=[], + help='''files and directories to include in the result. + If this is not specified, the current directory is used''') + parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files') + parser.add_argument('-f', '--ftypes', nargs='+', default=[], + help="list of file types to filter on. All files are included if this option is absent") + parser.add_argument('--ext-only', dest='extonly', action='store_true', default=False, + help='only use extension to determine file type') + parser.add_argument('--exclude', nargs='+', default=[], + help='list of files and directories to exclude from results, relative to repo root') + parser.add_argument('--extless-only', dest='extless_only', action='store_true', default=False, + help='only include extensionless files in output') + args = parser.parse_args() + listing = list_files(targets=args.targets, ftypes=args.ftypes, use_shebang=not args.extonly, + modified_only=args.modified, exclude=args.exclude, extless_only=args.extless_only) + for l in listing: + print(l) diff --git a/tools/run-mypy b/tools/run-mypy new file mode 100755 index 0000000..0f53043 --- /dev/null +++ b/tools/run-mypy @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +from __future__ import absolute_import +from __future__ import print_function + +import os +import sys +import argparse +import subprocess + +import lister +from typing import cast, Dict, List + +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +os.chdir(os.path.dirname(TOOLS_DIR)) + +sys.path.append(os.path.dirname(TOOLS_DIR)) + +exclude = """ +""".split() + +default_targets = ['zulip/zulip', + 'zulip/setup.py'] + +parser = argparse.ArgumentParser(description="Run mypy on files tracked by git.") +parser.add_argument('targets', nargs='*', default=default_targets, + help="""files and directories to include in the result. + If this is not specified, the current directory is used""") +parser.add_argument('-m', '--modified', action='store_true', default=False, help='list only modified files') +parser.add_argument('-a', '--all', dest='all', action='store_true', default=False, + help="""run mypy on all python files, ignoring the exclude list. + This is useful if you have to find out which files fail mypy check.""") +parser.add_argument('--no-disallow-untyped-defs', dest='disallow_untyped_defs', action='store_false', default=True, + help="""Don't throw errors when functions are not annotated""") +parser.add_argument('--scripts-only', dest='scripts_only', action='store_true', default=False, + help="""Only type check extensionless python scripts""") +parser.add_argument('--strict-optional', dest='strict_optional', action='store_true', default=False, + help="""Use the --strict-optional flag with mypy""") +parser.add_argument('--warn-unused-ignores', dest='warn_unused_ignores', action='store_true', default=False, + help="""Use the --warn-unused-ignores flag with mypy""") +parser.add_argument('--no-ignore-missing-imports', dest='ignore_missing_imports', action='store_false', default=True, + help="""Don't use the --ignore-missing-imports flag with mypy""") +parser.add_argument('--quick', action='store_true', default=False, + help="""Use the --quick flag with mypy""") +args = parser.parse_args() + +if args.all: + exclude = [] + +# find all non-excluded files in current directory +files_dict = cast(Dict[str, List[str]], + lister.list_files(targets=args.targets, ftypes=['py', 'pyi'], + use_shebang=True, modified_only=args.modified, + exclude = exclude + ['stubs'], group_by_ftype=True, + extless_only=args.scripts_only)) +pyi_files = set(files_dict['pyi']) +python_files = [fpath for fpath in files_dict['py'] + if not fpath.endswith('.py') or fpath + 'i' not in pyi_files] + +mypy_command = "mypy" + +extra_args = ["--check-untyped-defs", + "--follow-imports=silent", + "--scripts-are-modules", + "-i"] +if args.disallow_untyped_defs: + extra_args.append("--disallow-untyped-defs") +if args.warn_unused_ignores: + extra_args.append("--warn-unused-ignores") +if args.strict_optional: + extra_args.append("--strict-optional") +if args.ignore_missing_imports: + extra_args.append("--ignore-missing-imports") +if args.quick: + extra_args.append("--quick") + +# run mypy +if python_files: + rc = subprocess.call([mypy_command] + extra_args + python_files) + sys.exit(rc) +else: + print("There are no files to run mypy on.")