luo980/cpptrace
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve testing workflow (#12)

Browse Source
This commit is contained in:
Jeremy Rifkin 2023-07-20 21:53:44 -04:00 committed by GitHub
parent 73925368cc
commit c062bddd83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 153 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build.yml vendored
View File

@ -13,6 +13,7 @@ jobs:
run: sudo apt install gcc-10 g++-10 libgcc-10-dev
- name: build
run: |
pip3 install colorama
python3 ci/build-in-all-configs.py
build-macos:
runs-on: macos-13
@ -20,6 +21,7 @@ jobs:
- uses: actions/checkout@v2
- name: build
run: |
pip3 install colorama
python3 ci/build-in-all-configs.py
build-windows-msvc:
runs-on: windows-2019
@ -29,6 +31,7 @@ jobs:
uses: ilammy/msvc-dev-cmd@v1.10.0
- name: build
run: |
pip3 install colorama
python3 ci/build-in-all-configs.py --msvc-only
build-windows-clang:
runs-on: windows-2019
@ -38,4 +41,5 @@ jobs:
uses: ilammy/msvc-dev-cmd@v1.10.0
- name: build
run: |
pip3 install colorama
python3 ci/build-in-all-configs.py --clang-only

4
.github/workflows/performance-tests.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
- name: test
working-directory: build
run: |
./speedtest | python3 ../test/speedtest.py ${{matrix.config}}
./speedtest | python3 ../ci/speedtest.py ${{matrix.config}}
# TODO: For some reason this is slow on github's runner
#test-windows:
# runs-on: windows-2019
@ -73,4 +73,4 @@ jobs:
# - name: test
# working-directory: build
# run: |
# .\${{matrix.target}}\speedtest.exe | python3 ../test/speedtest.py ${{matrix.config}}
# .\${{matrix.target}}\speedtest.exe | python3 ../ci/speedtest.py ${{matrix.config}}

4
.github/workflows/test.yml vendored
View File

@ -15,6 +15,7 @@ jobs:
run: sudo apt install gcc-10 g++-10 libgcc-10-dev
- name: build
run: |
pip3 install colorama
python3 ci/test-all-configs.py
build-macos:
runs-on: macos-13
@ -22,6 +23,7 @@ jobs:
- uses: actions/checkout@v2
- name: build
run: |
pip3 install colorama
python3 ci/test-all-configs.py
build-windows-msvc:
runs-on: windows-2019
@ -31,6 +33,7 @@ jobs:
uses: ilammy/msvc-dev-cmd@v1.10.0
- name: build
run: |
pip3 install colorama
python3 ci/test-all-configs.py --msvc-only
build-windows-clang:
runs-on: windows-2019
@ -40,4 +43,5 @@ jobs:
uses: ilammy/msvc-dev-cmd@v1.10.0
- name: build
run: |
pip3 install colorama
python3 ci/test-all-configs.py --clang-only

11
ci/build-in-all-configs.py
View File

@ -4,6 +4,7 @@ import platform
import shutil
import subprocess
import sys
from colorama import Fore, Back, Style
from util import *
@ -12,12 +13,12 @@ sys.stdout.reconfigure(encoding='utf-8') # for windows gh runner
failed = False
def run_command(*args: List[str]):
print("[🔵 Running Command \"{}\"]".format(" ".join(args)))
print(f"{Fore.CYAN}{Style.BRIGHT}Running Command \"{' '.join(args)}\"{Style.RESET_ALL}")
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
print("\033[0m", end="") # makefile in parallel sometimes messes up colors
print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors
if p.returncode != 0:
print("[🔴 Command `{}` failed]".format(" ".join(args)))
print(f"{Fore.RED}{Style.BRIGHT}Command failed{Style.RESET_ALL}")
print("stdout:")
print(stdout.decode("utf-8"), end="")
print("stderr:")
@ -26,11 +27,11 @@ def run_command(*args: List[str]):
failed = True
return False
else:
print("[🟢 Command `{}` succeeded]".format(" ".join(args)))
print(f"{Fore.GREEN}{Style.BRIGHT}Command succeeded{Style.RESET_ALL}")
return True
def build(matrix):
print(matrix)
print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build with config {', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}")
if os.path.exists("build"):
shutil.rmtree("build")

0
test/speedtest.py → ci/speedtest.py
View File

200
ci/test-all-configs.py
View File

@ -4,6 +4,8 @@ import platform
import shutil
import subprocess
import sys
from typing import Tuple
from colorama import Fore, Back, Style
from util import *
@ -11,13 +13,95 @@ sys.stdout.reconfigure(encoding='utf-8') # for windows gh runner
failed = False
expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../test/expected/")
MAX_LINE_DIFF = 2
def similarity(name: str, target: List[str]) -> int:
parts = name.split(".txt")[0].split(".")
c = 0
for part in parts:
if part in target:
c += 1
else:
return -1
return c
def output_matches(output: str, params: Tuple[str]):
target = []
if params[0].startswith("gcc") or params[0].startswith("g++"):
target.append("gcc")
elif params[0].startswith("clang"):
target.append("clang")
elif params[0].startswith("cl"):
target.append("msvc")
if platform.system() == "Windows":
target.append("windows")
elif platform.system() == "Darwin":
target.append("macos")
else:
target.append("linux")
other_configs = params[1:]
for config in other_configs:
assert "WITH_" in config
target.append(config.split("WITH_")[1].lower())
print(f"Searching for expected file best matching {target}")
files = [f for f in os.listdir(expected_dir) if os.path.isfile(os.path.join(expected_dir, f))]
if len(files) == 0:
print(f"Error: No expected files to use (searching {expected_dir})")
sys.exit(1)
files = list(map(lambda f: (f, similarity(f, target)), files))
m = max(files, key=lambda entry: entry[1])[1]
if m <= 0:
print(f"Error: Could not find match for {target} in {files}")
sys.exit(1)
files = [entry[0] for entry in files if entry[1] == m]
if len(files) > 1:
print(f"Error: Ambiguous expected file to use ({files})")
sys.exit(1)
file = files[0]
print(f"Reading from {file}")
with open(os.path.join(expected_dir, file), "r") as f:
expected = f.read()
if output.strip() == "":
print(f"Error: No output from test")
sys.exit(1)
expected = [line.strip().split("||") for line in expected.split("\n")]
output = [line.strip().split("||") for line in output.split("\n")]
errored = False
for i, ((output_file, output_line, output_symbol), (expected_file, expected_line, expected_symbol)) in enumerate(zip(output, expected)):
if output_file != expected_file:
print(f"Error: File name mismatch on line {i + 1}, found \"{output_file}\" expected \"{expected_file}\"")
errored = True
if abs(int(output_line) - int(expected_line)) > MAX_LINE_DIFF:
print(f"Error: File line mismatch on line {i + 1}, found {output_line} expected {expected_line}")
errored = True
if output_symbol != expected_symbol:
print(f"Error: File symbol mismatch on line {i + 1}, found \"{output_symbol}\" expected \"{expected_symbol}\"")
errored = True
if expected_symbol == "main" or expected_symbol == "main()":
break
return not errored
def run_command(*args: List[str]):
print("[🔵 Running Command \"{}\"]".format(" ".join(args)))
print(f"{Fore.CYAN}{Style.BRIGHT}Running Command \"{' '.join(args)}\"{Style.RESET_ALL}")
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
print("\033[0m", end="") # makefile in parallel sometimes messes up colors
print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors
if p.returncode != 0:
print("[🔴 Command `{}` failed]".format(" ".join(args)))
print(f"{Fore.RED}{Style.BRIGHT}Command failed{Style.RESET_ALL}")
print("stdout:")
print(stdout.decode("utf-8"), end="")
print("stderr:")
@ -26,31 +110,31 @@ def run_command(*args: List[str]):
failed = True
return False
else:
print("[🟢 Command `{}` succeeded]".format(" ".join(args)))
print(f"{Fore.GREEN}{Style.BRIGHT}Command succeeded{Style.RESET_ALL}")
return True
def run_test(test_binary, *driver_args: List[str]):
print("[🔵 Running Command \"{} | {}\"]".format(test_binary, " ".join(driver_args)))
def run_test(test_binary, params: Tuple[str]):
print(f"{Fore.CYAN}{Style.BRIGHT}Running test{Style.RESET_ALL}")
test = subprocess.Popen([test_binary], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
driver = subprocess.Popen(driver_args, stdin=test.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
test.wait()
test_stderr = test.stderr.read()
driver_stdout, driver_stderr = driver.communicate()
print("\033[0m", end="") # makefile in parallel sometimes messes up colors
if test.returncode != 0 or driver.returncode != 0:
print("[🔴 Command `{} |{}` failed]".format(test_binary, " ".join(driver_args)))
print("test stderr:")
test_stdout, test_stderr = test.communicate()
print(Style.RESET_ALL, end="") # makefile in parallel sometimes messes up colors
if test.returncode != 0:
print("[🔴 Test command failed]")
print("stderr:")
print(test_stderr.decode("utf-8"), end="")
print("stdout:")
print(driver_stdout.decode("utf-8"), end="")
print(test_stdout.decode("utf-8"), end="")
else:
if len(test_stderr) != 0:
print("stderr:")
print(driver_stderr.decode("utf-8"), end="")
print(test_stderr.decode("utf-8"), end="")
if output_matches(test_stdout.decode("utf-8"), params):
print(f"{Fore.GREEN}{Style.BRIGHT}Test succeeded{Style.RESET_ALL}")
else:
print(f"{Fore.RED}{Style.BRIGHT}Test failed{Style.RESET_ALL}")
global failed
failed = True
return False
else:
print("[🟢 Command `{} | {}` succeeded]".format(test_binary, " ".join(driver_args)))
return True
def build(matrix):
if platform.system() != "Windows":
@ -88,43 +172,6 @@ def build(matrix):
if succeeded:
return run_command("msbuild", "cpptrace.sln")
def test(matrix):
if platform.system() != "Windows":
run_test(
"./test",
"python3",
"../test/test.py",
matrix["compiler"],
matrix["unwind"],
matrix["symbols"],
matrix["demangle"]
)
else:
run_test(
f".\\{matrix['target']}\\test.exe",
"python3",
"../test/test.py",
matrix["compiler"],
matrix["unwind"],
matrix["symbols"],
matrix["demangle"]
)
def build_and_test(matrix):
print(matrix)
if os.path.exists("build"):
shutil.rmtree("build")
os.mkdir("build")
os.chdir("build")
if build(matrix):
test(matrix)
os.chdir("..")
print()
def build_full_or_auto(matrix):
if platform.system() != "Windows":
args = [
@ -159,24 +206,47 @@ def build_full_or_auto(matrix):
if succeeded:
return run_command("msbuild", "cpptrace.sln")
def test_full_or_auto(matrix):
def test(matrix):
if platform.system() != "Windows":
run_test(
"./test",
"python3",
"../test/test.py",
matrix["compiler"]
(matrix["compiler"], matrix["unwind"], matrix["symbols"], matrix["demangle"])
)
else:
run_test(
f".\\{matrix['target']}\\test.exe",
"python3",
"../test/test.py",
matrix["compiler"]
(matrix["compiler"], matrix["unwind"], matrix["symbols"], matrix["demangle"])
)
def test_full_or_auto(matrix):
if platform.system() != "Windows":
run_test(
"./test",
(matrix["compiler"],)
)
else:
run_test(
f".\\{matrix['target']}\\test.exe",
(matrix["compiler"],)
)
def build_and_test(matrix):
print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build and test with config {', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}")
if os.path.exists("build"):
shutil.rmtree("build")
os.mkdir("build")
os.chdir("build")
if build(matrix):
test(matrix)
os.chdir("..")
print()
def build_and_test_full_or_auto(matrix):
print(matrix)
print(f"{Fore.BLUE}{Style.BRIGHT}{'=' * 10} Running build and test with config {'<auto>' if matrix['config'] == '' else ', '.join(matrix.values())} {'=' * 10}{Style.RESET_ALL}")
if os.path.exists("build"):
shutil.rmtree("build")

101
test/test.py
View File

@ -1,101 +0,0 @@
import os
import sys
import platform
from typing import List
MAX_LINE_DIFF = 2
def similarity(name: str, target: List[str]) -> int:
parts = name.split(".txt")[0].split(".")
c = 0
for part in parts:
if part in target:
c += 1
else:
return -1
return c
def main():
if len(sys.argv) < 2:
print("Expected at least one arg")
sys.exit(1)
target = []
if sys.argv[1].startswith("gcc") or sys.argv[1].startswith("g++"):
target.append("gcc")
elif sys.argv[1].startswith("clang"):
target.append("clang")
elif sys.argv[1].startswith("cl"):
target.append("msvc")
if platform.system() == "Windows":
target.append("windows")
elif platform.system() == "Darwin":
target.append("macos")
else:
target.append("linux")
other_configs = sys.argv[2:]
for config in other_configs:
assert "WITH_" in config
target.append(config.split("WITH_")[1].lower())
print(f"Searching for expected file best matching {target}")
expected_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected/")
files = [f for f in os.listdir(expected_dir) if os.path.isfile(os.path.join(expected_dir, f))]
if len(files) == 0:
print(f"Error: No expected files to use (searching {expected_dir})", file=sys.stderr)
sys.exit(1)
files = list(map(lambda f: (f, similarity(f, target)), files))
m = max(files, key=lambda entry: entry[1])[1]
if m <= 0:
print(f"Error: Could not find match for {target} in {files}", file=sys.stderr)
sys.exit(1)
files = [entry[0] for entry in files if entry[1] == m]
if len(files) > 1:
print(f"Error: Ambiguous expected file to use ({files})", file=sys.stderr)
sys.exit(1)
file = files[0]
print(f"Reading from {file}")
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "expected/", file), "r") as f:
expected = f.read()
output = sys.stdin.read()
print(output) # for debug reasons
if output.strip() == "":
print(f"Error: No output from test", file=sys.stderr)
sys.exit(1)
raw_output = output
expected = [line.split("||") for line in expected.split("\n")]
output = [line.split("||") for line in output.split("\n")]
errored = False
for i, ((output_file, output_line, output_symbol), (expected_file, expected_line, expected_symbol)) in enumerate(zip(output, expected)):
if output_file != expected_file:
print(f"Error: File name mismatch on line {i + 1}, found \"{output_file}\" expected \"{expected_file}\"", file=sys.stderr)
errored = True
if abs(int(output_line) - int(expected_line)) > MAX_LINE_DIFF:
print(f"Error: File line mismatch on line {i + 1}, found {output_line} expected {expected_line}", file=sys.stderr)
errored = True
if output_symbol != expected_symbol:
print(f"Error: File symbol mismatch on line {i + 1}, found \"{output_symbol}\" expected \"{expected_symbol}\"", file=sys.stderr)
errored = True
if expected_symbol == "main" or expected_symbol == "main()":
break
if errored:
print("Test failed")
sys.exit(1)
else:
print("Test passed")
main()