alpha
/
Kaleidoscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve `include-what-you-use` wrapper script

Browse Source 
This adds better argument parsing, and more useful options for detecting an
analyzing errors.  I relies on version >=0.18 of IWYU to work properly, because
prior to that its exit codes were non-standard and unhelpful.

Signed-off-by: Michael Richters <gedankenexperimenter@gmail.com>
pull/1158/head
Michael Richters 3 years ago
parent ea291858b2
commit dee60b0fce
No known key found for this signature in database
GPG Key ID: 1288FD13E4EEF0C0
2 changed files with 334 additions and 85 deletions
Show Stats Download Patch File Download Diff File

6
bin/fix-header-includes
Unescape View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
: "${KALEIDOSCOPE_DIR:=$(pwd)}"
cd "${KALEIDOSCOPE_DIR}" || exit 1
git ls-files -m | grep -E '\.(h|cpp)$' | xargs "${KALEIDOSCOPE_DIR}"/bin/iwyu.py

413
bin/iwyu.py
Unescape View File

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Copyright (c) 2022 Michael Richters <gedankenexperimenter@gmail.com> # Copyright (c) 2022 Michael Richters <gedankenexperimenter@gmail.com>
@ -28,7 +28,6 @@
# For more information, please refer to <http://unlicense.org/> # For more information, please refer to <http://unlicense.org/>
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
"""This is a script for maintenance of the headers included in Kaleidoscope source """This is a script for maintenance of the headers included in Kaleidoscope source
files. It is not currently possible to run this automatically on all files. It is not currently possible to run this automatically on all
Kaleidoscope source files, because of the peculiarities therein. It uses Kaleidoscope source files, because of the peculiarities therein. It uses
@ -46,49 +45,172 @@ made).
# Example invocation: # Example invocation:
# $ git ls-files -m | grep '\.\(h\|cpp\)' | bin/iwyu.py # $ git ls-files -m | grep '\.\(h\|cpp\)' | bin/iwyu.py
import argparse
import glob
import logging
import os import os
import re
import shlex
import shutil import shutil
import subprocess import subprocess
import sys import sys
# ==============================================================================
def parse_args(args):
parser = argparse.ArgumentParser(
description=
"""Run `include-what-you-use` on source files given as command-line arguments and/or read
from standard input. When reading target filenames from standard input, they should be
either absolute or relative to the current directory, and each line of input (minus the
line-ending character(s) is treated as a filename.""")
parser.add_argument(
'-q',
'--quiet',
dest='loglevel',
help="Suppress output except warnings and errors.",
action='store_const',
const=logging.ERROR,
default=logging.WARNING,
)
parser.add_argument(
'-v',
'--verbose',
dest='loglevel',
help="Output verbose debugging information.",
action='store_const',
const=logging.INFO,
)
parser.add_argument(
'-d',
'--debug',
dest='loglevel',
help="""Save output from `include-what-you-use` for processed files beside the
originals, with a '.iwyu' suffix, for debugging purposes.""",
action='store_const',
const=logging.DEBUG,
)
parser.add_argument(
'-r',
'--regex',
dest='regex',
help="""A regular expression for matching filenames Only the basename of the file is
matched, and the regex is only used when searching a directory for files to process,
not on target filenames specified in arguments or read from standard input.""",
action='store',
default=r'\.(h|cpp)$',
)
parser.add_argument(
'-i',
'--ignores_file',
dest='ignores_file',
metavar='<ignores_file>',
help=
"""The name of a file (relative to KALEIDOSCOPE_DIR) that contains a list of glob
patterns that will be ignored when a target directory is searched for filenames that
match <regex>.""",
action='store',
default='.iwyu_ignore',
)
parser.add_argument(
'targets',
metavar="<target>",
nargs='+',
help=
"""A list of target files and/or directories to search for source files to format. Any
target file will be processed, regardless of the filename. Any target directory will
be recursively searched for files matching the regular expression given by --regex.
Filenames and directories beginning with a '.' will always be excluded from the search,
but can still be processed if specified as a command-line target.""",
)
return parser.parse_args(args)
# ==============================================================================
def setup_logging(loglevel):
"""Set up basic logging
Args:
:int:loglevel: minimum loglevel for emitting messages
"""
logformat = "%(message)s"
logging.basicConfig(
level=loglevel,
stream=sys.stdout,
format=logformat,
datefmt="",
)
# ==============================================================================
def main(): def main():
"""Organize includes in Kaleidoscpe source files.""" """Main entry point function."""
# Parse command-line arguments:
opts = parse_args(sys.argv[1:])
# Set up logging system:
setup_logging(opts.loglevel)
# ----------------------------------------------------------------------
# Find include-what-you-use:
iwyu = shutil.which('include-what-you-use') iwyu = shutil.which('include-what-you-use')
print(f'IWYU: {iwyu}') logging.info("Found `include-what-you-use` executable: %s", iwyu)
iwyu_flags = [ iwyu_opts = [
'-Xiwyu', '--no_fwd_decls', # No forward declarations '--no_fwd_decls', # No forward declarations
'-x', 'c++', '--max_line_length=100',
'--update_comments',
] ]
# Prepend '-Xiwyu' to each `include-what-you-use` option:
iwyu_opts = [_ for opt in iwyu_opts for _ in ('-Xiwyu', opt)]
# ----------------------------------------------------------------------
# Find fix_includes:
fix_includes = shutil.which('fix_includes.py') fix_includes = shutil.which('fix_includes.py')
print(f'fix_includes: {fix_includes}') logging.debug("Found `fix_includes` executable: %s", fix_includes)
# ----------------------------------------------------------------------
# Find clang (first checking environment variable):
clang = os.getenv('CLANG_COMPILER') clang = os.getenv('CLANG_COMPILER')
if clang is None: if clang is None:
clang = shutil.which('clang') clang = shutil.which('clang')
print(f'clang: {clang}') logging.debug("Found `clang` executable: %s", clang)
result = subprocess.run([clang, '-print-resource-dir'], # Get system include dir from `clang`:
capture_output=True, check=True) clang_cmd = [clang, '-print-resource-dir']
logging.debug("Running command: `%s`", shlex.join(clang_cmd))
result = subprocess.run(clang_cmd, capture_output=True, check=True)
clang_resource_dir = result.stdout.decode('utf-8').rstrip() clang_resource_dir = result.stdout.decode('utf-8').rstrip()
system_include_dir = os.path.join(clang_resource_dir, 'include') system_include_dir = os.path.join(clang_resource_dir, 'include')
logging.debug("Using system include dir: %s", system_include_dir)
# ----------------------------------------------------------------------
# Get $KALEIDOSCOPE_DIR from enironment, falling back on `pwd`:
kaleidoscope_dir = os.getenv('KALEIDOSCOPE_DIR') kaleidoscope_dir = os.getenv('KALEIDOSCOPE_DIR')
if kaleidoscope_dir is None: if kaleidoscope_dir is None:
kaleidoscope_dir = os.getcwd() kaleidoscope_dir = os.getcwd()
logging.debug("Using Kaleidoscope dir: %s", kaleidoscope_dir)
kaleidoscope_src_dir = os.path.join(kaleidoscope_dir, 'src') kaleidoscope_src_dir = os.path.join(kaleidoscope_dir, 'src')
print(f'KALEIDOSCOPE_DIR: {kaleidoscope_dir}')
# Define locations of other dirs to find Arduino libraries:
virtual_hardware_dir = os.path.join( virtual_hardware_dir = os.path.join(
kaleidoscope_dir, '.arduino', 'user', 'hardware', 'keyboardio', 'virtual' kaleidoscope_dir, '.arduino', 'user', 'hardware', 'keyboardio', 'virtual')
) logging.debug("Using virtual hardware dir: %s", virtual_hardware_dir)
virtual_arduino_core_dir = os.path.join(virtual_hardware_dir, 'cores', 'arduino') virtual_arduino_core_dir = os.path.join(virtual_hardware_dir, 'cores', 'arduino')
logging.debug("Using virtual arduino core: %s", virtual_arduino_core_dir)
virtual_model01_dir = os.path.join(virtual_hardware_dir, 'variants', 'model01') virtual_model01_dir = os.path.join(virtual_hardware_dir, 'variants', 'model01')
virtual_keyboardiohid_dir = os.path.join(virtual_hardware_dir, logging.debug("Using virtual Model01 dir: %s", virtual_model01_dir)
'libraries', 'KeyboardioHID', 'src')
virtual_keyboardiohid_dir = os.path.join(
virtual_hardware_dir, 'libraries', 'KeyboardioHID', 'src')
logging.debug("Using virtual KeyboardioHID dir: %s", virtual_keyboardiohid_dir)
clang_flags = [ # ----------------------------------------------------------------------
# Create the long list of options passed to `clang` via `include-what-you-use`.
# First, we tell it we're using C++:
clang_opts = ['-x', 'c++']
# General compiler options:
clang_opts += [
'-c', '-c',
'-g', '-g',
'-Wall', '-Wall',
@ -103,76 +225,197 @@ def main():
'-Wno-unused-variable', '-Wno-unused-variable',
'-Wno-ignored-qualifiers', '-Wno-ignored-qualifiers',
'-Wno-type-limits', '-Wno-type-limits',
'-D' + 'KALEIDOSCOPE_VIRTUAL_BUILD=1', '-Wno-pragma-once-outside-header',
'-D' + 'KEYBOARDIOHID_BUILD_WITHOUT_HID=1', ]
'-D' + 'USBCON=dummy',
'-D' + 'ARDUINO_ARCH_AVR=1', # Variables we define to do a Kaleidoscope build:
'-D' + 'ARDUINO=10607', defines = [
'-D' + 'ARDUINO_AVR_MODEL01', 'KALEIDOSCOPE_VIRTUAL_BUILD=1',
'-D' + 'ARDUINO_ARCH_VIRTUAL', 'KEYBOARDIOHID_BUILD_WITHOUT_HID=1',
'-D' + 'USB_VID=0x1209', 'USBCON=dummy',
'-D' + 'USB_PID=0x2301', 'ARDUINO_ARCH_AVR=1',
'-D' + 'USB_MANUFACTURER="Keyboardio"', 'ARDUINO=10607',
'-D' + 'USB_PRODUCT="Model 01"', 'ARDUINO_AVR_MODEL01',
'-D' + 'KALEIDOSCOPE_HARDWARE_H="Kaleidoscope-Hardware-Keyboardio-Model01.h"', 'ARDUINO_ARCH_VIRTUAL',
'-D' + 'TWI_BUFFER_LENGTH=32', 'USB_VID=0x1209',
'-I' + system_include_dir, 'USB_PID=0x2301',
'-I' + kaleidoscope_src_dir, 'USB_MANUFACTURER="Keyboardio"',
'-I' + virtual_arduino_core_dir, 'USB_PRODUCT="Model 01"',
'-I' + virtual_model01_dir, 'KALEIDOSCOPE_HARDWARE_H="Kaleidoscope-Hardware-Keyboardio-Model01.h"',
'-I' + virtual_keyboardiohid_dir, 'TWI_BUFFER_LENGTH=32',
]
clang_opts += ['-D' + _ for _ in defines]
# Directories to search for libraries to include:
includes = [
system_include_dir,
kaleidoscope_src_dir,
virtual_arduino_core_dir,
virtual_model01_dir,
virtual_keyboardiohid_dir,
]
# Include plugin source dirs for plugins that depend on other plugins:
includes += glob.glob(os.path.join(kaleidoscope_dir, 'plugins', '*', 'src'))
clang_opts += ['-I' + _ for _ in includes]
# ----------------------------------------------------------------------
# Define the `include-what-you-use` command (sans target files)
iwyu_cmd = [iwyu] + iwyu_opts + clang_opts
logging.debug("Using IWYU command: %s", ' \\\n\t'.join(iwyu_cmd))
fix_includes_cmd = [
fix_includes,
'--update_comments',
'--nosafe_headers',
'--reorder',
'--separate_project_includes=' + kaleidoscope_src_dir, # Does this help?
] ]
logging.debug("Using `fix_includes` command: %s", ' \\\n\t'.join(fix_includes_cmd))
plugins_dir = os.path.join(kaleidoscope_dir, 'plugins') # ----------------------------------------------------------------------
for basename in os.listdir(plugins_dir): targets = opts.targets
plugin_dir = os.path.join(plugins_dir, basename) # If stdin is a pipe, read pathname targets, one per line. This allows us to
if not os.path.isdir(plugin_dir): # connect the output of `find` to our input conveniently:
if not sys.stdin.isatty():
targets += sys.stdin.read().splitlines()
# ----------------------------------------------------------------------
iwyu_ignores_file = os.path.join(kaleidoscope_dir, opts.ignores_file)
ignores = build_ignores_list(iwyu_ignores_file)
# ----------------------------------------------------------------------
regex = re.compile(opts.regex)
exit_code = 0
for src in (_ for t in targets for _ in build_target_list(t, regex)):
if src in ignores:
logging.info("Skipping ignored file: %s", os.path.relpath(src))
continue continue
clang_flags.append('-I' + os.path.join(plugin_dir, 'src')) if not run_iwyu(os.path.relpath(src), iwyu_cmd, fix_includes_cmd):
exit_code = 1
for arg in [iwyu] + iwyu_flags: return exit_code
print(arg)
for arg in clang_flags:
print(arg) # ==============================================================================
def build_target_list(path, src_regex):
for source_file in sys.argv[1:]: """Docstring"""
iwyu_cmd = [iwyu] + iwyu_flags + clang_flags + [source_file] logging.debug("Searching target: %s", path)
print('------------------------------------------------------------')
print(f'File: {source_file}') # If the specified path is a filename, return it (as a list), regardless of
# whether or not it matches the regex.
# Sometimes, it's useful to force IWYU to make changes, or to have a if os.path.isfile(path):
# more definitive marker of whether or not it failed due to compilation return [path]
# errors (which may differ between IWYU and normal compilation,
# unfortunately). If so, the follwing code can be uncommented. It adds # If the specified path is not valid, just return an empty list.
# a harmless `#include` at the end of the file, which will be removed if if not os.path.isdir(path):
# this script runs successfully. logging.error("Error: File not found: %s", path)
return []
# with open(source_file, "rb+") as fd:
# fd.seek(-1, 2) # The specified path is a directory, so we search recursively for files
# char = fd.read(1) # contained therein that match the specified regular expression.
# if char != b'\n': targets = []
# print('missing newline at end of file') for root, dirs, files in os.walk(os.path.abspath(path)):
# fd.write(b'\n') logging.debug("Searching dir: %s", root)
# if source_file != 'src/kaleidoscope/HIDTables.h': # First, ignore all dotfiles (and directories).
# fd.write(b'#include "kaleidoscope/HIDTables.h"') dotfiles = set(glob.glob('.*'))
dirs = set(dirs) - dotfiles
iwyu_proc = subprocess.run(iwyu_cmd, capture_output=True, check=False) files = set(files) - dotfiles
fix_includes_cmd = [ logging.debug("Files found: %s", ', '.join(files))
fix_includes, # Add all matching files to the list of source files to be formatted.
'--update_comments', for f in filter(src_regex.search, files):
'--nosafe_headers', logging.debug("Source file found: %s", f)
# Don't change the order of headers in existing files, because some targets.append(os.path.join(root, f))
# of them have been changed from what IWYU will do. For new files,
# use `--reorder` instead. return [os.path.abspath(_) for _ in targets]
'--noreorder',
]
subprocess.run(fix_includes_cmd, input=iwyu_proc.stderr, check=False) # ==============================================================================
def build_ignores_list(ignores_file_path):
# Optionally, we can write the output of `include-what-you-use` to a logging.debug("Searching for ignores file: %s", ignores_file_path)
# file for debugging purposes: # If the ignores file doesn't exist, return an empty list:
# with open(source_file + '.iwyu', 'wb') as fd: if not os.path.isfile(ignores_file_path):
# fd.write(iwyu_proc.stderr) logging.debug("Ignores file not found")
return []
ignores_list = []
with open(ignores_file_path) as f:
for line in f.read().splitlines():
logging.debug("Ignoring files like: %s", line)
if line.startswith('#'):
continue
ignores_list += glob.glob(line, recursive=True)
ignores_file_dir = os.path.dirname(ignores_file_path)
with cwd(ignores_file_dir):
ignores_list[:] = [os.path.abspath(_) for _ in ignores_list]
logging.debug("Ignores list:\n\t%s", "\n\t".join(ignores_list))
return ignores_list
# ------------------------------------------------------------------------------
from contextlib import contextmanager
@contextmanager
def cwd(path):
"""A simple function change directory, an automatically restore the previous working
directory when done, using `with cwd(temp_dir):`"""
old_wd = os.getcwd()
os.chdir(path)
try:
yield
finally:
os.chdir(old_wd)
# ==============================================================================
def run_iwyu(source_file, iwyu_cmd, fix_includes_cmd):
"""Run `include-what-you-use` on <source_file>, an update that file's header includes by
sending the output to `fix_includes.py`. If either command returns an error code, return
`False`, otherwise return `True`."""
logging.info("Processing file: %s", source_file)
# Run IWYU on <source_file>
iwyu_proc = subprocess.run(iwyu_cmd + [source_file], capture_output=True, check=False)
# If IWYU returns an error, report on it:
if iwyu_proc.returncode != 0:
logging.error("Error: failed to parse file: %s", source_file)
logging.debug("IWYU returned: %s", iwyu_proc.returncode)
logging.debug("STDOUT:\n%s", iwyu_proc.stdout.decode('utf-8'))
logging.debug("STDERR:\n%s", iwyu_proc.stderr.decode('utf-8'))
# In addition to reporting the error, save the output for analysis:
with open(source_file + '.iwyu', 'wb') as f:
f.write(iwyu_proc.stderr)
# Don't run fix_includes if there was an error (or if we've got an old version of IWYU
# that returns bogus exit codes):
return False
# IWYU reports on the associated header of *.cpp files, but if we want to skip processing
# that header, we need to use only the part of the output for the *.cpp file. Fortunately,
# the header is listed first, so we only need to search for the start of the target file's
# section of the output.
n = iwyu_proc.stderr.find(f"\n{source_file} should".encode('utf-8'))
iwyu_stderr = iwyu_proc.stderr[n:]
# Run fix_includes.py, using the output (stderr) of IWYU:
fix_includes_proc = subprocess.run(
fix_includes_cmd, input=iwyu_stderr, capture_output=True, check=False)
# Report any errors returned by fix_includes.py:
if fix_includes_proc.returncode != 0:
logging.error("Error: failed to fix includes for file: %s", source_file)
logging.debug("fix_includes.py returned: %s", fix_includes_proc.returncode)
logging.debug("STDOUT:\n%s", fix_includes_proc.stdout.decode('utf-8'))
logging.debug("STDERR:\n%s", fix_includes_proc.stderr.decode('utf-8'))
return False
# Return true on success, false otherwise:
return True
# ==============================================================================
if __name__ == "__main__": if __name__ == "__main__":
main() try:
sys.exit(main())
except KeyboardInterrupt:
logging.info("Aborting")
sys.exit(1)

Write Preview
Loading…
Cancel
Save