#!/usr/bin/env python # Copyright (c) 2021 Egor Tensin # This file is part of the "cmake-common" project. # For details, see https://github.com/egor-tensin/cmake-common. # Distributed under the MIT License. """Wrap your actual test driver to use with CTest CTest suffers from at least two issues, in particular with regard to its PASS_REGULAR_EXPRESSION feature: 1. The regular expression syntax used by CMake is deficient. 2. The exit code of a test is ignored if one of the regexes matches. This script tries to fix them. """ import argparse import os import re import subprocess import sys SCRIPT_NAME = os.path.basename(__file__) def dump(msg, **kwargs): print(f"{SCRIPT_NAME}: {msg}", **kwargs) def err(msg): dump(msg, file=sys.stderr) def read_file(path): with open(path, mode="r") as fd: return fd.read() def run(cmd_line, exit_codes=[0]): try: result = subprocess.run( cmd_line, check=True, universal_newlines=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, ) assert result.returncode == 0 return result.stdout except subprocess.CalledProcessError as e: if e.returncode not in exit_codes: err( f"Actual exit code {e.returncode} is not among the allowed codes {exit_codes}" ) sys.stdout.write(e.output) sys.exit(e.returncode) return e.output def run_new_window(cmd_line, exit_codes=[0]): try: result = subprocess.run( cmd_line, check=True, creationflags=subprocess.CREATE_NEW_CONSOLE ) assert result.returncode == 0 except subprocess.CalledProcessError as e: if e.returncode not in exit_codes: err( f"Actual exit code {e.returncode} is not among the allowed codes {exit_codes}" ) sys.exit(e.returncode) return None def match(s, regex): return re.search(regex, s, flags=re.MULTILINE) def match_any(s, regexes): return any([match(s, regex) for regex in regexes]) def match_all(s, regexes): return all([match(s, regex) for regex in regexes]) def match_pass_regexes(output, regexes): if not regexes: return if not match_all(output, regexes): err( "Couldn't match test program's output against all of the" " regular expressions:" ) for regex in regexes: err(f"\t{regex}") sys.exit(1) def match_fail_regexes(output, regexes): if not regexes: return if match_any(output, regexes): err("Matched test program's output against some of the regular" " expressions:") for regex in regexes: err(f"\t{regex}") sys.exit(1) def run_actual_test_driver(args): cmd_line = [args.exe_path] + args.exe_args run_func = run if args.new_window: run_func = run_new_window output = run_func(cmd_line, args.exit_codes) if args.new_window and (args.pass_regexes or args.fail_regexes): err("Cannot launch child process in a new window and capture its output") if output is not None: sys.stdout.write(output) match_pass_regexes(output, args.pass_regexes) match_fail_regexes(output, args.fail_regexes) def grep_file(args): contents = read_file(args.path) match_pass_regexes(contents, args.pass_regexes) match_fail_regexes(contents, args.fail_regexes) def parse_args(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) subparsers = parser.add_subparsers(dest="command") parser_run = subparsers.add_parser( "run", help="run an executable and check its output" ) parser_run.add_argument( "-p", "--pass-regex", nargs="*", dest="pass_regexes", metavar="REGEX", help="pass if all of these regexes match", ) parser_run.add_argument( "-f", "--fail-regex", nargs="*", dest="fail_regexes", metavar="REGEX", help="fail if any of these regexes matches", ) parser_run.add_argument( "-e", "--exit-code", nargs="*", type=int, default=[0], dest="exit_codes", metavar="NUM", help="allowed exit_codes (only 0 by default)", ) parser_run.add_argument( "-n", "--new-window", action="store_true", help="launch child process in a new console window", ) parser_run.add_argument( "exe_path", metavar="PATH", help="path to the test executable" ) # nargs='*' here would discard additional '--'s. parser_run.add_argument( "exe_args", metavar="ARG", nargs=argparse.REMAINDER, help="test executable arguments", ) parser_run.set_defaults(func=run_actual_test_driver) parser_grep = subparsers.add_parser( "grep", help="check file contents for matching patterns" ) parser_grep.add_argument( "-p", "--pass-regex", nargs="*", dest="pass_regexes", metavar="REGEX", help="pass if all of these regexes match", ) parser_grep.add_argument( "-f", "--fail-regex", nargs="*", dest="fail_regexes", metavar="REGEX", help="fail if any of these regexes matches", ) parser_grep.add_argument("path", metavar="PATH", help="text file path") parser_grep.set_defaults(func=grep_file) args = parser.parse_args(argv) if args.command is None: parser.error("please specify a subcommand to run") return args def main(argv=None): args = parse_args(argv) args.func(args) if __name__ == "__main__": main()