Improve testing workflow (#12)

.github/workflows/build.yml

@@ -13,6 +13,7 @@ jobs:
       run: sudo apt install gcc-10 g++-10 libgcc-10-dev
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/build-in-all-configs.py
   build-macos:
     runs-on: macos-13
@@ -20,6 +21,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/build-in-all-configs.py
   build-windows-msvc:
     runs-on: windows-2019
@@ -29,6 +31,7 @@ jobs:
       uses: ilammy/msvc-dev-cmd@v1.10.0
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/build-in-all-configs.py --msvc-only
   build-windows-clang:
     runs-on: windows-2019
@@ -38,4 +41,5 @@ jobs:
       uses: ilammy/msvc-dev-cmd@v1.10.0
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/build-in-all-configs.py --clang-only

.github/workflows/performance-tests.yml

@@ -41,7 +41,7 @@ jobs:
     - name: test
       working-directory: build
       run: |
-           ./speedtest | python3 ../test/speedtest.py ${{matrix.config}}
+           ./speedtest | python3 ../ci/speedtest.py ${{matrix.config}}
   # TODO: For some reason this is slow on github's runner
   #test-windows:
   #  runs-on: windows-2019
@@ -73,4 +73,4 @@ jobs:
   #  - name: test
   #    working-directory: build
   #    run: |
-  #         .\${{matrix.target}}\speedtest.exe | python3 ../test/speedtest.py ${{matrix.config}}
+  #         .\${{matrix.target}}\speedtest.exe | python3 ../ci/speedtest.py ${{matrix.config}}

.github/workflows/test.yml

@@ -15,6 +15,7 @@ jobs:
       run: sudo apt install gcc-10 g++-10 libgcc-10-dev
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/test-all-configs.py
   build-macos:
     runs-on: macos-13
@@ -22,6 +23,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/test-all-configs.py
   build-windows-msvc:
     runs-on: windows-2019
@@ -31,6 +33,7 @@ jobs:
       uses: ilammy/msvc-dev-cmd@v1.10.0
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/test-all-configs.py --msvc-only
   build-windows-clang:
     runs-on: windows-2019
@@ -40,4 +43,5 @@ jobs:
       uses: ilammy/msvc-dev-cmd@v1.10.0
     - name: build
       run: |
+           pip3 install colorama
            python3 ci/test-all-configs.py --clang-only

ci/build-in-all-configs.py

@@ -4,6 +4,7 @@ import platform
 import shutil
 import subprocess
 import sys
+from colorama import Fore, Back, Style
 
 from util import *
 
@@ -12,12 +13,12 @@ sys.stdout.reconfigure(encoding='utf-8') # for windows gh runner
 failed = False
 
 def run_command(*args: List[str]):
-    print("[🔵 Running Command \"{}\"]".format(" ".join(args)))
+    print(f"{Fore.CYAN}{Style.BRIGHT}Running Command \"{' '.join(args)}\"{Style.RESET_ALL}")
     p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
-    print("\033[0m", end="") # makefile in parallel sometimes messes up colors
+    print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors
     if p.returncode != 0:
-        print("[🔴 Command `{}` failed]".format(" ".join(args)))
+        print(f"{Fore.RED}{Style.BRIGHT}Command failed{Style.RESET_ALL}")
         print("stdout:")
         print(stdout.decode("utf-8"), end="")
         print("stderr:")
@@ -26,11 +27,11 @@ def run_command(*args: List[str]):
         failed = True
         return False
     else:
-        print("[🟢 Command `{}` succeeded]".format(" ".join(args)))
+        print(f"{Fore.GREEN}{Style.BRIGHT}Command succeeded{Style.RESET_ALL}")
         return True
 
 def build(matrix):
-    print(matrix)
+    print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build with config {', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}")
 
     if os.path.exists("build"):
         shutil.rmtree("build")

ci/test-all-configs.py

@@ -4,6 +4,8 @@ import platform
 import shutil
 import subprocess
 import sys
+from typing import Tuple
+from colorama import Fore, Back, Style
 
 from util import *
 
@@ -11,13 +13,95 @@ sys.stdout.reconfigure(encoding='utf-8') # for windows gh runner
 
 failed = False
 
+expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../test/expected/")
+
+MAX_LINE_DIFF = 2
+
+def similarity(name: str, target: List[str]) -> int:
+    parts = name.split(".txt")[0].split(".")
+    c = 0
+    for part in parts:
+        if part in target:
+            c += 1
+        else:
+            return -1
+    return c
+
+def output_matches(output: str, params: Tuple[str]):
+    target = []
+
+    if params[0].startswith("gcc") or params[0].startswith("g++"):
+        target.append("gcc")
+    elif params[0].startswith("clang"):
+        target.append("clang")
+    elif params[0].startswith("cl"):
+        target.append("msvc")
+
+    if platform.system() == "Windows":
+        target.append("windows")
+    elif platform.system() == "Darwin":
+        target.append("macos")
+    else:
+        target.append("linux")
+
+    other_configs = params[1:]
+    for config in other_configs:
+        assert "WITH_" in config
+        target.append(config.split("WITH_")[1].lower())
+
+    print(f"Searching for expected file best matching {target}")
+
+    files = [f for f in os.listdir(expected_dir) if os.path.isfile(os.path.join(expected_dir, f))]
+    if len(files) == 0:
+        print(f"Error: No expected files to use (searching {expected_dir})")
+        sys.exit(1)
+    files = list(map(lambda f: (f, similarity(f, target)), files))
+    m = max(files, key=lambda entry: entry[1])[1]
+    if m <= 0:
+        print(f"Error: Could not find match for {target} in {files}")
+        sys.exit(1)
+    files = [entry[0] for entry in files if entry[1] == m]
+    if len(files) > 1:
+        print(f"Error: Ambiguous expected file to use ({files})")
+        sys.exit(1)
+
+    file = files[0]
+    print(f"Reading from {file}")
+
+    with open(os.path.join(expected_dir, file), "r") as f:
+        expected = f.read()
+
+    if output.strip() == "":
+        print(f"Error: No output from test")
+        sys.exit(1)
+
+    expected = [line.strip().split("||") for line in expected.split("\n")]
+    output = [line.strip().split("||") for line in output.split("\n")]
+
+    errored = False
+
+    for i, ((output_file, output_line, output_symbol), (expected_file, expected_line, expected_symbol)) in enumerate(zip(output, expected)):
+        if output_file != expected_file:
+            print(f"Error: File name mismatch on line {i + 1}, found \"{output_file}\" expected \"{expected_file}\"")
+            errored = True
+        if abs(int(output_line) - int(expected_line)) > MAX_LINE_DIFF:
+            print(f"Error: File line mismatch on line {i + 1}, found {output_line} expected {expected_line}")
+            errored = True
+        if output_symbol != expected_symbol:
+            print(f"Error: File symbol mismatch on line {i + 1}, found \"{output_symbol}\" expected \"{expected_symbol}\"")
+            errored = True
+        if expected_symbol == "main" or expected_symbol == "main()":
+            break
+
+    return not errored
+
 def run_command(*args: List[str]):
-    print("[🔵 Running Command \"{}\"]".format(" ".join(args)))
+    print(f"{Fore.CYAN}{Style.BRIGHT}Running Command \"{' '.join(args)}\"{Style.RESET_ALL}")
     p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
-    print("\033[0m", end="") # makefile in parallel sometimes messes up colors
+    print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors
     if p.returncode != 0:
-        print("[🔴 Command `{}` failed]".format(" ".join(args)))
+        print(f"{Fore.RED}{Style.BRIGHT}Command failed{Style.RESET_ALL}")
         print("stdout:")
         print(stdout.decode("utf-8"), end="")
         print("stderr:")
@@ -26,31 +110,31 @@ def run_command(*args: List[str]):
         failed = True
         return False
     else:
-        print("[🟢 Command `{}` succeeded]".format(" ".join(args)))
+        print(f"{Fore.GREEN}{Style.BRIGHT}Command succeeded{Style.RESET_ALL}")
         return True
 
-def run_test(test_binary, *driver_args: List[str]):
+def run_test(test_binary, params: Tuple[str]):
-    print("[🔵 Running Command \"{} | {}\"]".format(test_binary, " ".join(driver_args)))
+    print(f"{Fore.CYAN}{Style.BRIGHT}Running test{Style.RESET_ALL}")
     test = subprocess.Popen([test_binary], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    driver = subprocess.Popen(driver_args, stdin=test.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    test_stdout, test_stderr = test.communicate()
-    test.wait()
+    print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors
-    test_stderr = test.stderr.read()
+
-    driver_stdout, driver_stderr = driver.communicate()
+    if test.returncode != 0:
-    print("\033[0m", end="") # makefile in parallel sometimes messes up colors
+        print("[🔴 Test command failed]")
-    if test.returncode != 0 or driver.returncode != 0:
+        print("stderr:")
-        print("[🔴 Command `{} |{}` failed]".format(test_binary, " ".join(driver_args)))
-        print("test stderr:")
         print(test_stderr.decode("utf-8"), end="")
         print("stdout:")
-        print(driver_stdout.decode("utf-8"), end="")
+        print(test_stdout.decode("utf-8"), end="")
-        print("stderr:")
-        print(driver_stderr.decode("utf-8"), end="")
-        global failed
-        failed = True
-        return False
     else:
-        print("[🟢 Command `{} | {}` succeeded]".format(test_binary, " ".join(driver_args)))
+        if len(test_stderr) != 0:
-        return True
+            print("stderr:")
+            print(test_stderr.decode("utf-8"), end="")
+        if output_matches(test_stdout.decode("utf-8"), params):
+            print(f"{Fore.GREEN}{Style.BRIGHT}Test succeeded{Style.RESET_ALL}")
+        else:
+            print(f"{Fore.RED}{Style.BRIGHT}Test failed{Style.RESET_ALL}")
+            global failed
+            failed = True
 
 def build(matrix):
     if platform.system() != "Windows":
@@ -88,43 +172,6 @@ def build(matrix):
         if succeeded:
             return run_command("msbuild", "cpptrace.sln")
 
-def test(matrix):
-    if platform.system() != "Windows":
-        run_test(
-            "./test",
-            "python3",
-            "../test/test.py",
-            matrix["compiler"],
-            matrix["unwind"],
-            matrix["symbols"],
-            matrix["demangle"]
-        )
-    else:
-        run_test(
-            f".\\{matrix['target']}\\test.exe",
-            "python3",
-            "../test/test.py",
-            matrix["compiler"],
-            matrix["unwind"],
-            matrix["symbols"],
-            matrix["demangle"]
-        )
-
-def build_and_test(matrix):
-    print(matrix)
-
-    if os.path.exists("build"):
-        shutil.rmtree("build")
-
-    os.mkdir("build")
-    os.chdir("build")
-
-    if build(matrix):
-        test(matrix)
-
-    os.chdir("..")
-    print()
-
 def build_full_or_auto(matrix):
     if platform.system() != "Windows":
         args = [
@@ -159,24 +206,47 @@ def build_full_or_auto(matrix):
         if succeeded:
             return run_command("msbuild", "cpptrace.sln")
 
-def test_full_or_auto(matrix):
+def test(matrix):
     if platform.system() != "Windows":
         run_test(
             "./test",
-            "python3",
+            (matrix["compiler"], matrix["unwind"], matrix["symbols"], matrix["demangle"])
-            "../test/test.py",
-            matrix["compiler"]
         )
     else:
         run_test(
             f".\\{matrix['target']}\\test.exe",
-            "python3",
+            (matrix["compiler"], matrix["unwind"], matrix["symbols"], matrix["demangle"])
-            "../test/test.py",
-            matrix["compiler"]
         )
 
+def test_full_or_auto(matrix):
+    if platform.system() != "Windows":
+        run_test(
+            "./test",
+            (matrix["compiler"],)
+        )
+    else:
+        run_test(
+            f".\\{matrix['target']}\\test.exe",
+            (matrix["compiler"],)
+        )
+
+def build_and_test(matrix):
+    print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build and test with config {', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}")
+
+    if os.path.exists("build"):
+        shutil.rmtree("build")
+
+    os.mkdir("build")
+    os.chdir("build")
+
+    if build(matrix):
+        test(matrix)
+
+    os.chdir("..")
+    print()
+
 def build_and_test_full_or_auto(matrix):
-    print(matrix)
+    print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build and test with config {'<auto>' if matrix['config'] == '' else ', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}")
 
     if os.path.exists("build"):
         shutil.rmtree("build")

test/test.py

@@ -1,101 +0,0 @@
-import os
-import sys
-import platform
-from typing import List
-
-MAX_LINE_DIFF = 2
-
-def similarity(name: str, target: List[str]) -> int:
-    parts = name.split(".txt")[0].split(".")
-    c = 0
-    for part in parts:
-        if part in target:
-            c += 1
-        else:
-            return -1
-    return c
-
-def main():
-    if len(sys.argv) < 2:
-        print("Expected at least one arg")
-        sys.exit(1)
-
-    target = []
-
-    if sys.argv[1].startswith("gcc") or sys.argv[1].startswith("g++"):
-        target.append("gcc")
-    elif sys.argv[1].startswith("clang"):
-        target.append("clang")
-    elif sys.argv[1].startswith("cl"):
-        target.append("msvc")
-
-    if platform.system() == "Windows":
-        target.append("windows")
-    elif platform.system() == "Darwin":
-        target.append("macos")
-    else:
-        target.append("linux")
-
-    other_configs = sys.argv[2:]
-    for config in other_configs:
-        assert "WITH_" in config
-        target.append(config.split("WITH_")[1].lower())
-
-    print(f"Searching for expected file best matching {target}")
-
-    expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected/")
-    files = [f for f in os.listdir(expected_dir) if os.path.isfile(os.path.join(expected_dir, f))]
-    if len(files) == 0:
-        print(f"Error: No expected files to use (searching {expected_dir})", file=sys.stderr)
-        sys.exit(1)
-    files = list(map(lambda f: (f, similarity(f, target)), files))
-    m = max(files, key=lambda entry: entry[1])[1]
-    if m <= 0:
-        print(f"Error: Could not find match for {target} in {files}", file=sys.stderr)
-        sys.exit(1)
-    files = [entry[0] for entry in files if entry[1] == m]
-    if len(files) > 1:
-        print(f"Error: Ambiguous expected file to use ({files})", file=sys.stderr)
-        sys.exit(1)
-
-    file = files[0]
-    print(f"Reading from {file}")
-
-    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected/", file), "r") as f:
-        expected = f.read()
-
-    output = sys.stdin.read()
-
-    print(output) # for debug reasons
-
-    if output.strip() == "":
-        print(f"Error: No output from test", file=sys.stderr)
-        sys.exit(1)
-
-    raw_output = output
-
-    expected = [line.split("||") for line in expected.split("\n")]
-    output = [line.split("||") for line in output.split("\n")]
-
-    errored = False
-
-    for i, ((output_file, output_line, output_symbol), (expected_file, expected_line, expected_symbol)) in enumerate(zip(output, expected)):
-        if output_file != expected_file:
-            print(f"Error: File name mismatch on line {i + 1}, found \"{output_file}\" expected \"{expected_file}\"", file=sys.stderr)
-            errored = True
-        if abs(int(output_line) - int(expected_line)) > MAX_LINE_DIFF:
-            print(f"Error: File line mismatch on line {i + 1}, found {output_line} expected {expected_line}", file=sys.stderr)
-            errored = True
-        if output_symbol != expected_symbol:
-            print(f"Error: File symbol mismatch on line {i + 1}, found \"{output_symbol}\" expected \"{expected_symbol}\"", file=sys.stderr)
-            errored = True
-        if expected_symbol == "main" or expected_symbol == "main()":
-            break
-
-    if errored:
-        print("Test failed")
-        sys.exit(1)
-    else:
-        print("Test passed")
-
-main()