diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e4761ff..561dbd3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,6 +13,7 @@ jobs: run: sudo apt install gcc-10 g++-10 libgcc-10-dev - name: build run: | + pip3 install colorama python3 ci/build-in-all-configs.py build-macos: runs-on: macos-13 @@ -20,6 +21,7 @@ jobs: - uses: actions/checkout@v2 - name: build run: | + pip3 install colorama python3 ci/build-in-all-configs.py build-windows-msvc: runs-on: windows-2019 @@ -29,6 +31,7 @@ jobs: uses: ilammy/msvc-dev-cmd@v1.10.0 - name: build run: | + pip3 install colorama python3 ci/build-in-all-configs.py --msvc-only build-windows-clang: runs-on: windows-2019 @@ -38,4 +41,5 @@ jobs: uses: ilammy/msvc-dev-cmd@v1.10.0 - name: build run: | + pip3 install colorama python3 ci/build-in-all-configs.py --clang-only diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 8151c58..ca1b525 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -41,7 +41,7 @@ jobs: - name: test working-directory: build run: | - ./speedtest | python3 ../test/speedtest.py ${{matrix.config}} + ./speedtest | python3 ../ci/speedtest.py ${{matrix.config}} # TODO: For some reason this is slow on github's runner #test-windows: # runs-on: windows-2019 @@ -73,4 +73,4 @@ jobs: # - name: test # working-directory: build # run: | - # .\${{matrix.target}}\speedtest.exe | python3 ../test/speedtest.py ${{matrix.config}} + # .\${{matrix.target}}\speedtest.exe | python3 ../ci/speedtest.py ${{matrix.config}} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbf9f63..84bf238 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ jobs: run: sudo apt install gcc-10 g++-10 libgcc-10-dev - name: build run: | + pip3 install colorama python3 ci/test-all-configs.py build-macos: runs-on: macos-13 @@ -22,6 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: build run: | + pip3 install colorama python3 ci/test-all-configs.py build-windows-msvc: runs-on: windows-2019 @@ -31,6 +33,7 @@ jobs: uses: ilammy/msvc-dev-cmd@v1.10.0 - name: build run: | + pip3 install colorama python3 ci/test-all-configs.py --msvc-only build-windows-clang: runs-on: windows-2019 @@ -40,4 +43,5 @@ jobs: uses: ilammy/msvc-dev-cmd@v1.10.0 - name: build run: | + pip3 install colorama python3 ci/test-all-configs.py --clang-only diff --git a/ci/build-in-all-configs.py b/ci/build-in-all-configs.py index 2cd4289..899aa0b 100644 --- a/ci/build-in-all-configs.py +++ b/ci/build-in-all-configs.py @@ -4,6 +4,7 @@ import platform import shutil import subprocess import sys +from colorama import Fore, Back, Style from util import * @@ -12,12 +13,12 @@ sys.stdout.reconfigure(encoding='utf-8') # for windows gh runner failed = False def run_command(*args: List[str]): - print("[🔵 Running Command \"{}\"]".format(" ".join(args))) + print(f"{Fore.CYAN}{Style.BRIGHT}Running Command \"{' '.join(args)}\"{Style.RESET_ALL}") p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() - print("\033[0m", end="") # makefile in parallel sometimes messes up colors + print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors if p.returncode != 0: - print("[🔴 Command `{}` failed]".format(" ".join(args))) + print(f"{Fore.RED}{Style.BRIGHT}Command failed{Style.RESET_ALL}") print("stdout:") print(stdout.decode("utf-8"), end="") print("stderr:") @@ -26,11 +27,11 @@ def run_command(*args: List[str]): failed = True return False else: - print("[🟢 Command `{}` succeeded]".format(" ".join(args))) + print(f"{Fore.GREEN}{Style.BRIGHT}Command succeeded{Style.RESET_ALL}") return True def build(matrix): - print(matrix) + print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build with config {', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}") if os.path.exists("build"): shutil.rmtree("build") diff --git a/test/speedtest.py b/ci/speedtest.py similarity index 100% rename from test/speedtest.py rename to ci/speedtest.py diff --git a/ci/test-all-configs.py b/ci/test-all-configs.py index ace7a26..9a3d3d4 100644 --- a/ci/test-all-configs.py +++ b/ci/test-all-configs.py @@ -4,6 +4,8 @@ import platform import shutil import subprocess import sys +from typing import Tuple +from colorama import Fore, Back, Style from util import * @@ -11,13 +13,95 @@ sys.stdout.reconfigure(encoding='utf-8') # for windows gh runner failed = False +expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../test/expected/") + +MAX_LINE_DIFF = 2 + +def similarity(name: str, target: List[str]) -> int: + parts = name.split(".txt")[0].split(".") + c = 0 + for part in parts: + if part in target: + c += 1 + else: + return -1 + return c + +def output_matches(output: str, params: Tuple[str]): + target = [] + + if params[0].startswith("gcc") or params[0].startswith("g++"): + target.append("gcc") + elif params[0].startswith("clang"): + target.append("clang") + elif params[0].startswith("cl"): + target.append("msvc") + + if platform.system() == "Windows": + target.append("windows") + elif platform.system() == "Darwin": + target.append("macos") + else: + target.append("linux") + + other_configs = params[1:] + for config in other_configs: + assert "WITH_" in config + target.append(config.split("WITH_")[1].lower()) + + print(f"Searching for expected file best matching {target}") + + files = [f for f in os.listdir(expected_dir) if os.path.isfile(os.path.join(expected_dir, f))] + if len(files) == 0: + print(f"Error: No expected files to use (searching {expected_dir})") + sys.exit(1) + files = list(map(lambda f: (f, similarity(f, target)), files)) + m = max(files, key=lambda entry: entry[1])[1] + if m <= 0: + print(f"Error: Could not find match for {target} in {files}") + sys.exit(1) + files = [entry[0] for entry in files if entry[1] == m] + if len(files) > 1: + print(f"Error: Ambiguous expected file to use ({files})") + sys.exit(1) + + file = files[0] + print(f"Reading from {file}") + + with open(os.path.join(expected_dir, file), "r") as f: + expected = f.read() + + if output.strip() == "": + print(f"Error: No output from test") + sys.exit(1) + + expected = [line.strip().split("||") for line in expected.split("\n")] + output = [line.strip().split("||") for line in output.split("\n")] + + errored = False + + for i, ((output_file, output_line, output_symbol), (expected_file, expected_line, expected_symbol)) in enumerate(zip(output, expected)): + if output_file != expected_file: + print(f"Error: File name mismatch on line {i + 1}, found \"{output_file}\" expected \"{expected_file}\"") + errored = True + if abs(int(output_line) - int(expected_line)) > MAX_LINE_DIFF: + print(f"Error: File line mismatch on line {i + 1}, found {output_line} expected {expected_line}") + errored = True + if output_symbol != expected_symbol: + print(f"Error: File symbol mismatch on line {i + 1}, found \"{output_symbol}\" expected \"{expected_symbol}\"") + errored = True + if expected_symbol == "main" or expected_symbol == "main()": + break + + return not errored + def run_command(*args: List[str]): - print("[🔵 Running Command \"{}\"]".format(" ".join(args))) + print(f"{Fore.CYAN}{Style.BRIGHT}Running Command \"{' '.join(args)}\"{Style.RESET_ALL}") p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() - print("\033[0m", end="") # makefile in parallel sometimes messes up colors + print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors if p.returncode != 0: - print("[🔴 Command `{}` failed]".format(" ".join(args))) + print(f"{Fore.RED}{Style.BRIGHT}Command failed{Style.RESET_ALL}") print("stdout:") print(stdout.decode("utf-8"), end="") print("stderr:") @@ -26,31 +110,31 @@ def run_command(*args: List[str]): failed = True return False else: - print("[🟢 Command `{}` succeeded]".format(" ".join(args))) + print(f"{Fore.GREEN}{Style.BRIGHT}Command succeeded{Style.RESET_ALL}") return True -def run_test(test_binary, *driver_args: List[str]): - print("[🔵 Running Command \"{} | {}\"]".format(test_binary, " ".join(driver_args))) +def run_test(test_binary, params: Tuple[str]): + print(f"{Fore.CYAN}{Style.BRIGHT}Running test{Style.RESET_ALL}") test = subprocess.Popen([test_binary], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - driver = subprocess.Popen(driver_args, stdin=test.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - test.wait() - test_stderr = test.stderr.read() - driver_stdout, driver_stderr = driver.communicate() - print("\033[0m", end="") # makefile in parallel sometimes messes up colors - if test.returncode != 0 or driver.returncode != 0: - print("[🔴 Command `{} |{}` failed]".format(test_binary, " ".join(driver_args))) - print("test stderr:") + test_stdout, test_stderr = test.communicate() + print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors + + if test.returncode != 0: + print("[🔴 Test command failed]") + print("stderr:") print(test_stderr.decode("utf-8"), end="") print("stdout:") - print(driver_stdout.decode("utf-8"), end="") - print("stderr:") - print(driver_stderr.decode("utf-8"), end="") - global failed - failed = True - return False + print(test_stdout.decode("utf-8"), end="") else: - print("[🟢 Command `{} | {}` succeeded]".format(test_binary, " ".join(driver_args))) - return True + if len(test_stderr) != 0: + print("stderr:") + print(test_stderr.decode("utf-8"), end="") + if output_matches(test_stdout.decode("utf-8"), params): + print(f"{Fore.GREEN}{Style.BRIGHT}Test succeeded{Style.RESET_ALL}") + else: + print(f"{Fore.RED}{Style.BRIGHT}Test failed{Style.RESET_ALL}") + global failed + failed = True def build(matrix): if platform.system() != "Windows": @@ -88,43 +172,6 @@ def build(matrix): if succeeded: return run_command("msbuild", "cpptrace.sln") -def test(matrix): - if platform.system() != "Windows": - run_test( - "./test", - "python3", - "../test/test.py", - matrix["compiler"], - matrix["unwind"], - matrix["symbols"], - matrix["demangle"] - ) - else: - run_test( - f".\\{matrix['target']}\\test.exe", - "python3", - "../test/test.py", - matrix["compiler"], - matrix["unwind"], - matrix["symbols"], - matrix["demangle"] - ) - -def build_and_test(matrix): - print(matrix) - - if os.path.exists("build"): - shutil.rmtree("build") - - os.mkdir("build") - os.chdir("build") - - if build(matrix): - test(matrix) - - os.chdir("..") - print() - def build_full_or_auto(matrix): if platform.system() != "Windows": args = [ @@ -159,24 +206,47 @@ def build_full_or_auto(matrix): if succeeded: return run_command("msbuild", "cpptrace.sln") -def test_full_or_auto(matrix): +def test(matrix): if platform.system() != "Windows": run_test( "./test", - "python3", - "../test/test.py", - matrix["compiler"] + (matrix["compiler"], matrix["unwind"], matrix["symbols"], matrix["demangle"]) ) else: run_test( f".\\{matrix['target']}\\test.exe", - "python3", - "../test/test.py", - matrix["compiler"] + (matrix["compiler"], matrix["unwind"], matrix["symbols"], matrix["demangle"]) ) +def test_full_or_auto(matrix): + if platform.system() != "Windows": + run_test( + "./test", + (matrix["compiler"],) + ) + else: + run_test( + f".\\{matrix['target']}\\test.exe", + (matrix["compiler"],) + ) + +def build_and_test(matrix): + print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build and test with config {', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}") + + if os.path.exists("build"): + shutil.rmtree("build") + + os.mkdir("build") + os.chdir("build") + + if build(matrix): + test(matrix) + + os.chdir("..") + print() + def build_and_test_full_or_auto(matrix): - print(matrix) + print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build and test with config {'' if matrix['config'] == '' else ', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}") if os.path.exists("build"): shutil.rmtree("build") diff --git a/test/test.py b/test/test.py deleted file mode 100644 index dd0cf6b..0000000 --- a/test/test.py +++ /dev/null @@ -1,101 +0,0 @@ -import os -import sys -import platform -from typing import List - -MAX_LINE_DIFF = 2 - -def similarity(name: str, target: List[str]) -> int: - parts = name.split(".txt")[0].split(".") - c = 0 - for part in parts: - if part in target: - c += 1 - else: - return -1 - return c - -def main(): - if len(sys.argv) < 2: - print("Expected at least one arg") - sys.exit(1) - - target = [] - - if sys.argv[1].startswith("gcc") or sys.argv[1].startswith("g++"): - target.append("gcc") - elif sys.argv[1].startswith("clang"): - target.append("clang") - elif sys.argv[1].startswith("cl"): - target.append("msvc") - - if platform.system() == "Windows": - target.append("windows") - elif platform.system() == "Darwin": - target.append("macos") - else: - target.append("linux") - - other_configs = sys.argv[2:] - for config in other_configs: - assert "WITH_" in config - target.append(config.split("WITH_")[1].lower()) - - print(f"Searching for expected file best matching {target}") - - expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected/") - files = [f for f in os.listdir(expected_dir) if os.path.isfile(os.path.join(expected_dir, f))] - if len(files) == 0: - print(f"Error: No expected files to use (searching {expected_dir})", file=sys.stderr) - sys.exit(1) - files = list(map(lambda f: (f, similarity(f, target)), files)) - m = max(files, key=lambda entry: entry[1])[1] - if m <= 0: - print(f"Error: Could not find match for {target} in {files}", file=sys.stderr) - sys.exit(1) - files = [entry[0] for entry in files if entry[1] == m] - if len(files) > 1: - print(f"Error: Ambiguous expected file to use ({files})", file=sys.stderr) - sys.exit(1) - - file = files[0] - print(f"Reading from {file}") - - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected/", file), "r") as f: - expected = f.read() - - output = sys.stdin.read() - - print(output) # for debug reasons - - if output.strip() == "": - print(f"Error: No output from test", file=sys.stderr) - sys.exit(1) - - raw_output = output - - expected = [line.split("||") for line in expected.split("\n")] - output = [line.split("||") for line in output.split("\n")] - - errored = False - - for i, ((output_file, output_line, output_symbol), (expected_file, expected_line, expected_symbol)) in enumerate(zip(output, expected)): - if output_file != expected_file: - print(f"Error: File name mismatch on line {i + 1}, found \"{output_file}\" expected \"{expected_file}\"", file=sys.stderr) - errored = True - if abs(int(output_line) - int(expected_line)) > MAX_LINE_DIFF: - print(f"Error: File line mismatch on line {i + 1}, found {output_line} expected {expected_line}", file=sys.stderr) - errored = True - if output_symbol != expected_symbol: - print(f"Error: File symbol mismatch on line {i + 1}, found \"{output_symbol}\" expected \"{expected_symbol}\"", file=sys.stderr) - errored = True - if expected_symbol == "main" or expected_symbol == "main()": - break - - if errored: - print("Test failed") - sys.exit(1) - else: - print("Test passed") - -main()