pass extra metadata (repo refs, timestamps) to test results
This commit is contained in:
Ivan Avdeev
@@ -20,7 +20,12 @@ build() {
|
||||
|
||||
rendertest() {
|
||||
pushd /build/HLRTest/render
|
||||
WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py --xash-dir /opt/hl run
|
||||
WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py \
|
||||
--xash-dir /opt/hl \
|
||||
--xash-revision "${XASH3D_REVISION}" \
|
||||
--pbr-revision "${PBR_REVISION}" \
|
||||
--tests-revision "${HLRTEST_REVISION}" \
|
||||
run
|
||||
popd
|
||||
}
|
||||
|
||||
|
||||
+38
-5
@@ -3,7 +3,7 @@ set -eux
|
||||
|
||||
NAME=xash-builder
|
||||
|
||||
HLRTEST_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
HLRTEST_REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
|
||||
source .env
|
||||
|
||||
@@ -20,10 +20,37 @@ build() {
|
||||
build-image
|
||||
}
|
||||
|
||||
git_short() {
|
||||
local dir="${1:-.}"
|
||||
local hash dirty_flag
|
||||
|
||||
# Get short commit hash
|
||||
hash=$(cd "$dir" && git rev-parse --short HEAD 2>/dev/null)
|
||||
|
||||
if [[ -z "$hash" ]]; then
|
||||
echo "Not a git repository" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check for uncommitted changes (staged or unstaged)
|
||||
dirty_flag=$(cd "$dir" && git status --porcelain --untracked-files=no 2>/dev/null)
|
||||
|
||||
if [[ -n "$dirty_flag" ]]; then
|
||||
echo "${hash}-dirty"
|
||||
else
|
||||
echo "$hash"
|
||||
fi
|
||||
}
|
||||
|
||||
render-test() {
|
||||
OUTPUT="${HLRTEST_PATH}/render/work"
|
||||
# Make sure the image ubuntu user can write to the output
|
||||
chmod -R o+rw "${OUTPUT}"
|
||||
local OUTPUT="${HLRTEST_REPO_DIR}/render/work"
|
||||
|
||||
local HLRTEST_REVISION=$(git_short "${HLRTEST_REPO_DIR}")
|
||||
local XASH3D_REVISION=$(git_short "${XASH3D_RT_REPO_DIR}")
|
||||
local PBR_REVISION=$(git_short "${HALFLIFE_PBR_REPO_DIR}")
|
||||
|
||||
# Make sure the image's ubuntu (uid=1000) user can write to the output
|
||||
#chmod -R o+rw "${OUTPUT}"
|
||||
|
||||
podman run -it --rm \
|
||||
--name ${NAME} \
|
||||
@@ -34,14 +61,20 @@ render-test() {
|
||||
--read-only \
|
||||
--tmpfs /tmp \
|
||||
\
|
||||
-e HLRTEST_REVISION="${HLRTEST_REVISION}" \
|
||||
-e XASH3D_REVISION="${XASH3D_REVISION}" \
|
||||
-e PBR_REVISION="${PBR_REVISION}" \
|
||||
\
|
||||
-v /dev/dri/renderD128:/dev/dri/renderD128:ro \
|
||||
\
|
||||
-v /opt/hl \
|
||||
-v /home/ubuntu \
|
||||
-v ${HALFLIFE_PBR_REPO_DIR}/valve/pbr:/opt/hl/valve/pbr:ro \
|
||||
-v ${HALFLIFE_PBR_REPO_DIR}/valve/bluenoise:/opt/hl/valve/bluenoise:ro \
|
||||
\
|
||||
-v ${XASH3D_RT_REPO_DIR}:/build/xash3d-fwgs:O \
|
||||
\
|
||||
-v ${HLRTEST_PATH}:/build/HLRTest:ro \
|
||||
-v ${HLRTEST_REPO_DIR}:/build/HLRTest:ro \
|
||||
-v ${OUTPUT}:/build/HLRTest/render/work \
|
||||
\
|
||||
ubuntu-xash-builder:latest \
|
||||
|
||||
+40
-32
@@ -114,7 +114,7 @@ function isPassed(value) {
|
||||
];
|
||||
}
|
||||
|
||||
function makeTable(fields, data, attrs_func) {
|
||||
function makeTable(fields, tests, attrs_func) {
|
||||
const table = Tag('table', null, null, [
|
||||
Tag('tr', null, null, (() => {
|
||||
let tds = [];
|
||||
@@ -126,8 +126,8 @@ function makeTable(fields, data, attrs_func) {
|
||||
]);
|
||||
|
||||
|
||||
for (const di in data) {
|
||||
let d = data[di];
|
||||
for (const di in tests) {
|
||||
let d = tests[di];
|
||||
const attrs = attrs_func ? attrs_func(d) : null;
|
||||
table.appendChild(Tag('tr', attrs, null, (() => {
|
||||
let ret = [];
|
||||
@@ -159,7 +159,7 @@ function makeTable(fields, data, attrs_func) {
|
||||
return table;
|
||||
}
|
||||
|
||||
function buildTestResultsTable(data) {
|
||||
function buildTestResultsTable(tests) {
|
||||
const fields = [
|
||||
{label: 'Test', field: 'test', tags_func: sectionLink},
|
||||
{label: 'Channel', field: 'channel', tags_func: channelWithEmoji},
|
||||
@@ -167,27 +167,37 @@ function buildTestResultsTable(data) {
|
||||
{label: 'Passed', field: 'fail', tags_func: isPassed},
|
||||
];
|
||||
|
||||
return [makeTable(fields, data, rowAttribs)];
|
||||
return [makeTable(fields, tests, rowAttribs)];
|
||||
}
|
||||
|
||||
// Filter out all success
|
||||
function filterData(data) {
|
||||
return data
|
||||
.filter((d) => { return d.diff_pct !== 0; })
|
||||
.sort((l, r) => {
|
||||
return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM
|
||||
});
|
||||
function filterTests(tests) {
|
||||
return tests
|
||||
.filter((t) => { return t.diff_pct !== 0; })
|
||||
.sort((l, r) => {
|
||||
return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM
|
||||
});
|
||||
}
|
||||
|
||||
function buildSummary(data) {
|
||||
const total = data.length;
|
||||
const fails = data.filter((d) => { return !!d.fail; }).length;
|
||||
function buildSummary(test_results) {
|
||||
const tests = test_results.tests
|
||||
const total = tests.length;
|
||||
const fails = tests.filter((d) => { return !!d.fail; }).length;
|
||||
const fails_pct = fails * 100.0 / total;
|
||||
return emojiToTag(`🧪 Tests 💥: ${fails}/${total}🏁 (⚠${fails_pct.toFixed(3)}%)`);
|
||||
const meta = test_results.metadata;
|
||||
return [
|
||||
Tag('ul', null, null, [
|
||||
Tag('li', null, `Test date: ${meta.time}`),
|
||||
// TODO add links to repos at specified revisions
|
||||
Tag('li', null, `Xash3D-FWGS revision: ${meta.revisions.xash3d}`),
|
||||
Tag('li', null, `Half-Life-PBR revision: ${meta.revisions.pbr}`),
|
||||
Tag('li', null, `HLRTest revision: ${meta.revisions.tests}`),
|
||||
Tag('li', null, `Rendering took: ${meta.phase_render_time_sec.toFixed(1)} seconds`)
|
||||
])].concat(emojiToTag(`🧪 Tests 💥: ${fails}/${total}🏁 (⚠${fails_pct.toFixed(3)}%)`));
|
||||
}
|
||||
|
||||
function buildTestResultImages(data) {
|
||||
return data.flatMap((d) => {
|
||||
function buildTestResultImages(tests) {
|
||||
return tests.flatMap((d) => {
|
||||
//return Tag('details', {id: makeId(d)}, null, [
|
||||
return Tag('div', {id: makeId(d), "class": "meta-block"}, null, [
|
||||
//Tag('summary', null, `${d.test}/${d.channel} δ=${d.diff_pct}`),
|
||||
@@ -207,10 +217,8 @@ function buildTestResultImages(data) {
|
||||
|
||||
|
||||
|
||||
function buildData(table, images, data, sort, filter, exact_match) {
|
||||
if (sort) {
|
||||
data = filterData(data);
|
||||
}
|
||||
function buildData(table, images, test_results, sort, filter, exact_match) {
|
||||
let tests = sort ? filterTests(test_results.tests) : test_results.tests
|
||||
if (filter) {
|
||||
let test = "", channel = "";
|
||||
const args = filter.split(" ");
|
||||
@@ -221,7 +229,7 @@ function buildData(table, images, data, sort, filter, exact_match) {
|
||||
test = filter;
|
||||
channel = filter;
|
||||
}
|
||||
data = data.filter((d) => {
|
||||
tests = tests.filter((d) => {
|
||||
const test_result = exact_match ? d.test === test : d.test.includes(test);
|
||||
const channel_result = exact_match ? d.channel === channel : d.channel.includes(channel);
|
||||
if (args.length > 1) {
|
||||
@@ -232,8 +240,8 @@ function buildData(table, images, data, sort, filter, exact_match) {
|
||||
});
|
||||
}
|
||||
|
||||
table.replaceChildren(...buildTestResultsTable(data));
|
||||
images.replaceChildren(...buildTestResultImages(data));
|
||||
table.replaceChildren(...buildTestResultsTable(tests));
|
||||
images.replaceChildren(...buildTestResultImages(tests));
|
||||
|
||||
for (let block of images.querySelectorAll(".image-container")) {
|
||||
block.addEventListener("click", function() {
|
||||
@@ -255,8 +263,8 @@ function buildData(table, images, data, sort, filter, exact_match) {
|
||||
|
||||
saveToLocalStorage("rendertest_tablesort", sort);
|
||||
}
|
||||
const buildDataSlowMode = debounce((table, images, data, sort, filter, exact_match) => {
|
||||
buildData(table, images, data, sort, filter, exact_match);
|
||||
const buildDataSlowMode = debounce((table, images, test_results, sort, filter, exact_match) => {
|
||||
buildData(table, images, test_results, sort, filter, exact_match);
|
||||
}, ()=>{}, 250);
|
||||
|
||||
|
||||
@@ -550,7 +558,7 @@ window.onload = () => {
|
||||
]),
|
||||
Tag("label", null, null, [
|
||||
Tag("input", {"type": "radio", "name": "sort_table", "value": "yes", ...uglyChecked(tableSort)}, null, null, "input", (e) => {
|
||||
buildData(table, images, data, true, filter.value, exactmatch_input.checked);
|
||||
buildData(table, images, TEST_RESULTS, true, filter.value, exactmatch_input.checked);
|
||||
// TODO: rewrite use reactive programming
|
||||
tableSort = true;
|
||||
}),
|
||||
@@ -558,7 +566,7 @@ window.onload = () => {
|
||||
]),
|
||||
Tag("label", null, null, [
|
||||
Tag("input", {"type": "radio", "name": "sort_table", "value": "no", ...uglyChecked(!tableSort)}, null, null, "input", (e) => {
|
||||
buildData(table, images, data, false, filter.value, exactmatch_input.checked);
|
||||
buildData(table, images, TEST_RESULTS, false, filter.value, exactmatch_input.checked);
|
||||
tableSort = false;
|
||||
}),
|
||||
Text(" no")
|
||||
@@ -570,12 +578,12 @@ window.onload = () => {
|
||||
Tag("label", {"class": "filter sticky"}, "Filter", [
|
||||
filter = Tag("input", {"type": "input", "title": "Hotkey: ALT+F", "name": "filter", "value": filter_value}, null, null, "input", (e) => {
|
||||
saveToLocalStorage("rendertest_filter", e.target.value);
|
||||
buildDataSlowMode(table, images, data, tableSort, e.target.value, exactmatch_input.checked);
|
||||
buildDataSlowMode(table, images, TEST_RESULTS, tableSort, e.target.value, exactmatch_input.checked);
|
||||
}),
|
||||
Tag("label", null, null, [
|
||||
exactmatch_input = Tag("input", {"type": "checkbox", "name": "exactmatch", ...uglyChecked(exactMatch)}, null, null, "change", (e) => {
|
||||
saveToLocalStorage("rendertest_exactmatch", e.target.checked);
|
||||
buildDataSlowMode(table, images, data, tableSort, filter.value, exactmatch_input.checked);
|
||||
buildDataSlowMode(table, images, TEST_RESULTS, tableSort, filter.value, exactmatch_input.checked);
|
||||
}),
|
||||
Text("exact match")
|
||||
])
|
||||
@@ -586,7 +594,7 @@ window.onload = () => {
|
||||
Tag("div", {"class": "content"}, null, [
|
||||
Tag("h1", null, "Rendertest report"),
|
||||
Tag("h2", null, "Summary"),
|
||||
Tag("div", {"id": "summary"}, null, [...buildSummary(data)]),
|
||||
Tag("div", {"id": "summary"}, null, buildSummary(TEST_RESULTS)),
|
||||
Tag("h2", null, "Images of things that are not perfect ", [
|
||||
Tag("span", {"class": "emoji"}, "⤵")
|
||||
]),
|
||||
@@ -595,7 +603,7 @@ window.onload = () => {
|
||||
])
|
||||
document.body.appendChild(gridContainer);
|
||||
|
||||
buildData(table, images, data, tableSort, filter_value, exactmatch_input.checked);
|
||||
buildData(table, images, TEST_RESULTS, tableSort, filter_value, exactmatch_input.checked);
|
||||
|
||||
// TODO: remove this
|
||||
if (sidebarPos === "right") {
|
||||
|
||||
+42
-8
@@ -2,12 +2,14 @@
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
imagecompare = f'{ROOT}/imagecompare'
|
||||
@@ -15,11 +17,11 @@ WORKDIR = f'{ROOT}/work'
|
||||
REPORT_ROOT = f'{ROOT}' # FIXME should be workdir?
|
||||
|
||||
def get_imagemagick_convert_cmd() -> str:
|
||||
if shutil.which("magick"):
|
||||
return "magick"
|
||||
if shutil.which("convert"):
|
||||
return "convert"
|
||||
raise RuntimeError("No ImageMagick binary found (tried: magick, convert)")
|
||||
if shutil.which("magick"):
|
||||
return "magick"
|
||||
if shutil.which("convert"):
|
||||
return "convert"
|
||||
raise RuntimeError("No ImageMagick binary found (tried: magick, convert)")
|
||||
convert = get_imagemagick_convert_cmd()
|
||||
|
||||
# TODO rename to <map>_<description>_<issue(opt)>
|
||||
@@ -66,10 +68,28 @@ parser.add_argument('--tests', '-t', type=test_list, default=saves, help='Run on
|
||||
# TODO how to check that the dir is valid? presence of xash3d executable and valve dir?
|
||||
parser.add_argument('--xash-dir', '-x', type=str, default=os.getcwd(), help='Path to xash3d-fwgs installation directory')
|
||||
|
||||
parser.add_argument('--xash-revision', type=str, default='unknown', help='xash engine repo revision, commit_id<-dirty>')
|
||||
parser.add_argument('--pbr-revision', type=str, default='unknown', help='PBR textures repo revision, commit_id<-dirty>')
|
||||
parser.add_argument('--tests-revision', type=str, default='unknown', help='HLRTest (this) repo revision, commit_id<-dirty>')
|
||||
|
||||
# TODO parse commands in type=.. function
|
||||
parser.add_argument('command', type=str, default=None, help='Action to perform')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
metadata = {}
|
||||
metadata['time'] = datetime.now().astimezone().isoformat()
|
||||
metadata['revisions'] = {
|
||||
'xash3d': args.xash_revision,
|
||||
'pbr': args.pbr_revision,
|
||||
'tests': args.tests_revision,
|
||||
}
|
||||
# TODO:
|
||||
# - OS/kernel version
|
||||
# - GPU model
|
||||
# - Vulkan version
|
||||
# - GPU driver version
|
||||
|
||||
def make_script(file, tests: [str]):
|
||||
header = '''sv_cheats 1
|
||||
developer 0
|
||||
@@ -132,6 +152,7 @@ def render():
|
||||
env = os.environ.copy()
|
||||
env['LD_LIBRARY_PATH'] = f'{args.xash_dir}'
|
||||
|
||||
start_time_sec = time.perf_counter()
|
||||
with open(f'{WORKDIR}/xash-stdout.log', 'wb') as stdout, open(f'{WORKDIR}/xash-stderr.log', 'wb') as stderr:
|
||||
result = subprocess.run([f'{args.xash_dir}/xash3d', '-ref', 'vk',
|
||||
'-nowriteconfig', '-nosound', '-log',
|
||||
@@ -139,6 +160,8 @@ def render():
|
||||
'-width', '1280', '-height', '800',
|
||||
'+exec', 'rendertest.script'],
|
||||
env=env, check=True, stdout=stdout, stderr=stderr)
|
||||
end_time_sec = time.perf_counter()
|
||||
metadata['phase_render_time_sec'] = end_time_sec - start_time_sec
|
||||
|
||||
def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
|
||||
result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
|
||||
@@ -165,6 +188,7 @@ def compare():
|
||||
diffs = []
|
||||
command_png()
|
||||
print(f'Comparing...')
|
||||
start_time_sec = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
for test in args.tests:
|
||||
for channel, _ in channels.items():
|
||||
@@ -175,11 +199,19 @@ def compare():
|
||||
|
||||
diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff))
|
||||
|
||||
results = [diff.result() for diff in diffs]
|
||||
tests = [diff.result() for diff in diffs]
|
||||
end_time_sec = time.perf_counter()
|
||||
metadata['phase_compare_time_sec'] = end_time_sec - start_time_sec
|
||||
|
||||
results = {
|
||||
'metadata': metadata,
|
||||
'tests': tests,
|
||||
}
|
||||
|
||||
json.dump(results, open(f'{WORKDIR}/test_results.json', 'w'))
|
||||
jsons = json.dumps(results)
|
||||
with open(f'{WORKDIR}/data.js', 'w') as js:
|
||||
js.write(f'"use strict";\nconst data = {jsons};\n')
|
||||
with open(f'{WORKDIR}/results.js', 'w') as js:
|
||||
js.write(f'"use strict";\nconst TEST_RESULTS = {jsons};\n')
|
||||
|
||||
def command_compare():
|
||||
compile()
|
||||
@@ -215,6 +247,8 @@ def command_render():
|
||||
copy_assets()
|
||||
render()
|
||||
|
||||
# TODO make one long sequence of functions, and then call enabled ones, passing state via files
|
||||
|
||||
# TODO dict
|
||||
match args.command:
|
||||
case 'run':
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
<link rel="stylesheet alternate" type="text/css" href="theme-light.css" disabled="true" data-theme="theme-light" title="☀️ Light theme">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<script src="utils.js"></script>
|
||||
<script src="work/data.js"></script>
|
||||
<script src="work/results.js"></script>
|
||||
<script src="index.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
Reference in New Issue
Block a user