diff options
author | Martin Braun <martin.braun@ettus.com> | 2022-11-17 12:45:50 +0100 |
---|---|---|
committer | Aki Tomita <121511582+atomita-ni@users.noreply.github.com> | 2023-08-07 15:35:56 -0500 |
commit | 0cc18f7e8b9d9100c4b8cef366fce91fc2c245b9 (patch) | |
tree | dce360a43302180059e3a365b3833f64ae48767e | |
parent | Modify files for treatment with clang-format (diff) | |
download | uhd-0cc18f7e8b9d9100c4b8cef366fce91fc2c245b9.tar.xz uhd-0cc18f7e8b9d9100c4b8cef366fce91fc2c245b9.zip |
tools: Add clang-formatting tools
This adds two tools to the ./tools/ subdirectory:
== clang-formatter.sh ==
This is simply a small shell script that can be executed from the top of
the UHD repository, and it will format all files according the the
.clang-format file. It can be executed as such:
$ CLANG_FORMAT=clang-format-14 ./tools/clang-formatter.sh apply
Specifying a clang-format executable is optional, but note that
clang-format 14.0 should be used.
== run-clang-format.py ==
This is a Python script that is a modified version from
https://github.com/gnuradio/clang-format-lint-action/blob/ \
0b0cb14cf220a070d2a8b2610bd74ad1546252a1/run-clang-format.py
It was modified to add --patch-file option.
Alongside this file is a .clang-format-ignore file, which is sourced
from this script. The command can be run as such:
$ ./tools/run-clang-format.py \
--clang-format-executable clang-format-14 \
--extensions c,cpp,h,hpp,hpp.in,ipp \
-r \
--patch-file format.patch \
/path/to/uhd-repo
It will provide both a nice output summary as well as a patch file that
can be consumed with `patch -p0 < format.patch`.
-rw-r--r-- | .clang-format-ignore | 18 | ||||
-rwxr-xr-x | tools/clang-formatter.sh | 48 | ||||
-rw-r--r-- | tools/run-clang-format.py | 420 |
3 files changed, 486 insertions, 0 deletions
diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 000000000..fe34bfbf9 --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1,18 @@ +./host/cmake +./host/lib/deps +./mpm/lib/mykonos +./mpm/lib/rfdc +./mpm/include/mpm/rfdc +./mpm/tools +./fpga +./tools +./firmware +*cdecode.* +*getopt.* +*_generated.h +*template_lvbitx.* +# Ignore all the junk +.git +.vscode +./host/build +.ccls-cache diff --git a/tools/clang-formatter.sh b/tools/clang-formatter.sh new file mode 100755 index 000000000..ee8843c8f --- /dev/null +++ b/tools/clang-formatter.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +cformat=${CLANG_FORMAT:-} +if [ -z "$cformat" ]; then + cformat=$(which clang-format) +fi + +_cmd=$1 +if [ -z "$_cmd" ]; then + _cmd="apply" +fi + +cformat_args="" +case $_cmd in + apply) + cformat_args="-i" + ;; + check) + cformat_args="-Werror --dry-run" + ;; + check_apply) + cformat_args="-Werror -i" + ;; + *) + echo "Usage: $0 [check|apply|check_apply]" +esac + +find . \ + -path './host/lib/deps' -prune -o \ + -path './host/cmake' -prune -o \ + -path './fpga' -prune -o \ + -path './firmware' -prune -o \ + -path './mpm/lib/mykonos' -prune -o \ + -path './mpm/lib/rfdc' -prune -o \ + -path './mpm/include/mpm/rfdc' -prune -o \ + -path './mpm/tools' -prune -o \ + -path './tools' -prune -o \ + -name "getopt.*" -prune -o \ + -name "cdecode.*" -prune -o \ + -name "*_generated.h" -prune -o \ + -name "*template_lvbitx.*" -prune -o \ + -name "*.cpp" -print -o \ + -name "*.hpp" -print -o \ + -name "*.cpp.in" -print -o \ + -name "*.hpp.in" -print -o \ + -name "*.ipp" -print -o \ + -name "*.c" -print -o \ + -name "*.h" -print | xargs -n 10 -P 2 $cformat $cformat_args diff --git a/tools/run-clang-format.py b/tools/run-clang-format.py new file mode 100644 index 000000000..a53bc41ff --- /dev/null +++ b/tools/run-clang-format.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +"""A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +MIT License + +Copyright (c) 2019 Slobodan Kletnikov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from __future__ import print_function, unicode_literals + +import argparse +import difflib +import fnmatch +import io +import errno +from functools import partial +import multiprocessing +import os +import signal +import subprocess +from subprocess import DEVNULL # py3k +import sys +import traceback + +DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx' +DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + +def excludes_from_file(ignore_file): + excludes = [] + try: + with io.open(ignore_file, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('#'): + # ignore comments + continue + pattern = line.rstrip() + if not pattern: + # allow empty lines + continue + excludes.append(pattern) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + return excludes + +def list_files(files, recursive=False, extensions=None, exclude=None): + extensions = extensions or [] + exclude = exclude or [] + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [ + x for x in dnames + if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) \ + and not pattern in os.path.join(dirpath, x) + ] + fpaths = [ + x for x in fpaths if not fnmatch.fnmatch(x, pattern) + ] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, + reformatted, + fromfile='{}\t(original)'.format(file), + tofile='{}\t(reformatted)'.format(file), + n=3)) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super(DiffError, self).__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super(UnexpectedError, self).__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, filename): + try: + return run_clang_format_diff(args, filename) + except DiffError: + raise + except Exception as e: + raise UnexpectedError( + '{}: {}: {}'.format(filename, e.__class__.__name__, e), e) + + +def run_clang_format_diff(args, filename): + try: + with io.open(filename, 'r', encoding='utf-8') as f: + original = f.readlines() + except IOError as exc: + raise DiffError(str(exc)) + invocation = [args.clang_format_executable, filename] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + # + # It's not pretty, due to Python 2 & 3 compatibility. + try: + proc = subprocess.Popen( + invocation, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + **{'encoding': 'utf-8'}) + except OSError as exc: + raise DiffError( + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(invocation), exc + ) + ) + proc_stdout = proc.stdout + proc_stderr = proc.stderr + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + return make_diff(filename, original, outs), errs, filename + + +def bold_red(s): + return '\x1b[1m\x1b[31m' + s + '\x1b[0m' + + +def colorize(diff_lines): + def bold(s): + return '\x1b[1m' + s + '\x1b[0m' + + def cyan(s): + return '\x1b[36m' + s + '\x1b[0m' + + def green(s): + return '\x1b[32m' + s + '\x1b[0m' + + def red(s): + return '\x1b[31m' + s + '\x1b[0m' + + for line in diff_lines: + if line[:4] in ['--- ', '+++ ']: + yield bold(line) + elif line.startswith('@@ '): + yield cyan(line) + elif line.startswith('+'): + yield green(line) + elif line.startswith('-'): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = 'error:' + if use_colors: + error_text = bold_red(error_text) + print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) + + +def parse_args(): + """ + Parse command line arguments + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--clang-format-executable', + metavar='EXECUTABLE', + help='path to the clang-format executable', + default='clang-format') + parser.add_argument( + '--extensions', + help='comma separated list of file extensions (default: {})'.format( + DEFAULT_EXTENSIONS), + default=DEFAULT_EXTENSIONS) + parser.add_argument( + '-r', + '--recursive', + action='store_true', + help='run recursively over directories') + parser.add_argument('files', metavar='file', nargs='+') + parser.add_argument( + '-q', + '--quiet', + action='store_true', + help="disable output, useful for the exit code") + parser.add_argument( + '-j', + metavar='N', + type=int, + default=0, + help='run N clang-format jobs in parallel' + ' (default number of cpus + 1)') + parser.add_argument( + '--color', + default='auto', + choices=['auto', 'always', 'never'], + help='show colored diff (default: auto)') + parser.add_argument( + '-e', + '--exclude', + metavar='PATTERN', + action='append', + default=[], + help='exclude paths matching the given glob-like pattern(s)' + ' from recursive search') + parser.add_argument( + '--patch-file', + help="Additionally store all diffs to this patch file.") + return parser.parse_args(), parser.prog + +def config_signal_handling(): + """ + use default signal handling, like diff return SIGINT value on ^C + https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + """ + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +def get_color_mode(args): + """ + Return a pair of bools: (colored_stdout, colored_stderr) + """ + if args.color == 'always': + return True, True + if args.color == 'auto': + return sys.stdout.isatty(), sys.stderr.isatty() + return False, False + +def check_linter(args, prog, use_colors): + """ + Make sure clang-format is installed, exit otherwise + """ + version_invocation = [args.clang_format_executable, str("--version")] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(prog, str(e), use_colors=use_colors) + exit(ExitStatus.TROUBLE) + except OSError as e: + print_trouble( + prog, + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(version_invocation), e + ), + use_colors=use_colors, + ) + exit(ExitStatus.TROUBLE) + +def get_exclude_paths(args): + """ + Return a list of excluded paths + """ + excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) + for exclude_list in args.exclude: + excludes.extend(exclude_list.split(",")) + return excludes + + +def main(): + """ + Go, go, go! + """ + config_signal_handling() + args, prog = parse_args() + colored_stdout, colored_stderr = get_color_mode(args) + check_linter(args, prog, colored_stderr) + + retcode = ExitStatus.SUCCESS + + excludes = get_exclude_paths(args) + + patch_file = None + if args.patch_file: + patch_file = open(args.patch_file, 'w') + + files = list_files( + args.files, + recursive=args.recursive, + exclude=excludes, + extensions=args.extensions.split(',')) + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered( + partial(run_clang_format_diff_wrapper, args), files) + broken_files = [] + while True: + try: + outs, errs, filename = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + broken_files.append(filename) + print_diff(outs, use_color=colored_stdout) + if patch_file: + patch_file.writelines(outs) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + if broken_files: + print("The following files have formatting issues:") + for broken_file in broken_files: + print("* {}".format(broken_file)) + if patch_file: + patch_file.close() + return retcode + + +if __name__ == '__main__': + sys.exit(main()) |