From 75fa9cae18c265d9449e5b6ff64759880c7ab4db Mon Sep 17 00:00:00 2001 From: Ivan Avdeev Date: Thu, 9 Apr 2026 15:24:54 -0400 Subject: [PATCH] pass extra metadata (repo refs, timestamps) to test results --- container/build-and-test.sh | 7 +++- container/run.sh | 43 +++++++++++++++++++--- render/index.js | 72 ++++++++++++++++++++----------------- render/rendertest.py | 50 +++++++++++++++++++++----- render/report.html | 2 +- 5 files changed, 127 insertions(+), 47 deletions(-) diff --git a/container/build-and-test.sh b/container/build-and-test.sh index 7f2b99b..b9a5d02 100755 --- a/container/build-and-test.sh +++ b/container/build-and-test.sh @@ -20,7 +20,12 @@ build() { rendertest() { pushd /build/HLRTest/render - WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py --xash-dir /opt/hl run + WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py \ + --xash-dir /opt/hl \ + --xash-revision "${XASH3D_REVISION}" \ + --pbr-revision "${PBR_REVISION}" \ + --tests-revision "${HLRTEST_REVISION}" \ + run popd } diff --git a/container/run.sh b/container/run.sh index 69e1974..7f5c10a 100755 --- a/container/run.sh +++ b/container/run.sh @@ -3,7 +3,7 @@ set -eux NAME=xash-builder -HLRTEST_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" +HLRTEST_REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" source .env @@ -20,10 +20,37 @@ build() { build-image } +git_short() { + local dir="${1:-.}" + local hash dirty_flag + + # Get short commit hash + hash=$(cd "$dir" && git rev-parse --short HEAD 2>/dev/null) + + if [[ -z "$hash" ]]; then + echo "Not a git repository" >&2 + return 1 + fi + + # Check for uncommitted changes (staged or unstaged) + dirty_flag=$(cd "$dir" && git status --porcelain --untracked-files=no 2>/dev/null) + + if [[ -n "$dirty_flag" ]]; then + echo "${hash}-dirty" + else + echo "$hash" + fi +} + render-test() { - OUTPUT="${HLRTEST_PATH}/render/work" - # Make sure the image ubuntu user can write to the output - chmod -R o+rw "${OUTPUT}" + local OUTPUT="${HLRTEST_REPO_DIR}/render/work" + + local HLRTEST_REVISION=$(git_short "${HLRTEST_REPO_DIR}") + local XASH3D_REVISION=$(git_short "${XASH3D_RT_REPO_DIR}") + local PBR_REVISION=$(git_short "${HALFLIFE_PBR_REPO_DIR}") + + # Make sure the image's ubuntu (uid=1000) user can write to the output + #chmod -R o+rw "${OUTPUT}" podman run -it --rm \ --name ${NAME} \ @@ -34,14 +61,20 @@ render-test() { --read-only \ --tmpfs /tmp \ \ + -e HLRTEST_REVISION="${HLRTEST_REVISION}" \ + -e XASH3D_REVISION="${XASH3D_REVISION}" \ + -e PBR_REVISION="${PBR_REVISION}" \ + \ -v /dev/dri/renderD128:/dev/dri/renderD128:ro \ \ -v /opt/hl \ + -v /home/ubuntu \ -v ${HALFLIFE_PBR_REPO_DIR}/valve/pbr:/opt/hl/valve/pbr:ro \ + -v ${HALFLIFE_PBR_REPO_DIR}/valve/bluenoise:/opt/hl/valve/bluenoise:ro \ \ -v ${XASH3D_RT_REPO_DIR}:/build/xash3d-fwgs:O \ \ - -v ${HLRTEST_PATH}:/build/HLRTest:ro \ + -v ${HLRTEST_REPO_DIR}:/build/HLRTest:ro \ -v ${OUTPUT}:/build/HLRTest/render/work \ \ ubuntu-xash-builder:latest \ diff --git a/render/index.js b/render/index.js index b58a2cc..7d1fdf6 100644 --- a/render/index.js +++ b/render/index.js @@ -114,7 +114,7 @@ function isPassed(value) { ]; } -function makeTable(fields, data, attrs_func) { +function makeTable(fields, tests, attrs_func) { const table = Tag('table', null, null, [ Tag('tr', null, null, (() => { let tds = []; @@ -126,8 +126,8 @@ function makeTable(fields, data, attrs_func) { ]); - for (const di in data) { - let d = data[di]; + for (const di in tests) { + let d = tests[di]; const attrs = attrs_func ? attrs_func(d) : null; table.appendChild(Tag('tr', attrs, null, (() => { let ret = []; @@ -159,7 +159,7 @@ function makeTable(fields, data, attrs_func) { return table; } -function buildTestResultsTable(data) { +function buildTestResultsTable(tests) { const fields = [ {label: 'Test', field: 'test', tags_func: sectionLink}, {label: 'Channel', field: 'channel', tags_func: channelWithEmoji}, @@ -167,27 +167,37 @@ function buildTestResultsTable(data) { {label: 'Passed', field: 'fail', tags_func: isPassed}, ]; - return [makeTable(fields, data, rowAttribs)]; + return [makeTable(fields, tests, rowAttribs)]; } // Filter out all success -function filterData(data) { - return data - .filter((d) => { return d.diff_pct !== 0; }) - .sort((l, r) => { - return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM - }); +function filterTests(tests) { + return tests + .filter((t) => { return t.diff_pct !== 0; }) + .sort((l, r) => { + return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM + }); } -function buildSummary(data) { - const total = data.length; - const fails = data.filter((d) => { return !!d.fail; }).length; +function buildSummary(test_results) { + const tests = test_results.tests + const total = tests.length; + const fails = tests.filter((d) => { return !!d.fail; }).length; const fails_pct = fails * 100.0 / total; - return emojiToTag(`๐Ÿงช Tests ๐Ÿ’ฅ: ${fails}/${total}๐Ÿ (โš ${fails_pct.toFixed(3)}%)`); + const meta = test_results.metadata; + return [ + Tag('ul', null, null, [ + Tag('li', null, `Test date: ${meta.time}`), + // TODO add links to repos at specified revisions + Tag('li', null, `Xash3D-FWGS revision: ${meta.revisions.xash3d}`), + Tag('li', null, `Half-Life-PBR revision: ${meta.revisions.pbr}`), + Tag('li', null, `HLRTest revision: ${meta.revisions.tests}`), + Tag('li', null, `Rendering took: ${meta.phase_render_time_sec.toFixed(1)} seconds`) + ])].concat(emojiToTag(`๐Ÿงช Tests ๐Ÿ’ฅ: ${fails}/${total}๐Ÿ (โš ${fails_pct.toFixed(3)}%)`)); } -function buildTestResultImages(data) { - return data.flatMap((d) => { +function buildTestResultImages(tests) { + return tests.flatMap((d) => { //return Tag('details', {id: makeId(d)}, null, [ return Tag('div', {id: makeId(d), "class": "meta-block"}, null, [ //Tag('summary', null, `${d.test}/${d.channel} ฮด=${d.diff_pct}`), @@ -207,10 +217,8 @@ function buildTestResultImages(data) { -function buildData(table, images, data, sort, filter, exact_match) { - if (sort) { - data = filterData(data); - } +function buildData(table, images, test_results, sort, filter, exact_match) { + let tests = sort ? filterTests(test_results.tests) : test_results.tests if (filter) { let test = "", channel = ""; const args = filter.split(" "); @@ -221,7 +229,7 @@ function buildData(table, images, data, sort, filter, exact_match) { test = filter; channel = filter; } - data = data.filter((d) => { + tests = tests.filter((d) => { const test_result = exact_match ? d.test === test : d.test.includes(test); const channel_result = exact_match ? d.channel === channel : d.channel.includes(channel); if (args.length > 1) { @@ -232,8 +240,8 @@ function buildData(table, images, data, sort, filter, exact_match) { }); } - table.replaceChildren(...buildTestResultsTable(data)); - images.replaceChildren(...buildTestResultImages(data)); + table.replaceChildren(...buildTestResultsTable(tests)); + images.replaceChildren(...buildTestResultImages(tests)); for (let block of images.querySelectorAll(".image-container")) { block.addEventListener("click", function() { @@ -255,8 +263,8 @@ function buildData(table, images, data, sort, filter, exact_match) { saveToLocalStorage("rendertest_tablesort", sort); } -const buildDataSlowMode = debounce((table, images, data, sort, filter, exact_match) => { - buildData(table, images, data, sort, filter, exact_match); +const buildDataSlowMode = debounce((table, images, test_results, sort, filter, exact_match) => { + buildData(table, images, test_results, sort, filter, exact_match); }, ()=>{}, 250); @@ -550,7 +558,7 @@ window.onload = () => { ]), Tag("label", null, null, [ Tag("input", {"type": "radio", "name": "sort_table", "value": "yes", ...uglyChecked(tableSort)}, null, null, "input", (e) => { - buildData(table, images, data, true, filter.value, exactmatch_input.checked); + buildData(table, images, TEST_RESULTS, true, filter.value, exactmatch_input.checked); // TODO: rewrite use reactive programming tableSort = true; }), @@ -558,7 +566,7 @@ window.onload = () => { ]), Tag("label", null, null, [ Tag("input", {"type": "radio", "name": "sort_table", "value": "no", ...uglyChecked(!tableSort)}, null, null, "input", (e) => { - buildData(table, images, data, false, filter.value, exactmatch_input.checked); + buildData(table, images, TEST_RESULTS, false, filter.value, exactmatch_input.checked); tableSort = false; }), Text(" no") @@ -570,12 +578,12 @@ window.onload = () => { Tag("label", {"class": "filter sticky"}, "Filter", [ filter = Tag("input", {"type": "input", "title": "Hotkey: ALT+F", "name": "filter", "value": filter_value}, null, null, "input", (e) => { saveToLocalStorage("rendertest_filter", e.target.value); - buildDataSlowMode(table, images, data, tableSort, e.target.value, exactmatch_input.checked); + buildDataSlowMode(table, images, TEST_RESULTS, tableSort, e.target.value, exactmatch_input.checked); }), Tag("label", null, null, [ exactmatch_input = Tag("input", {"type": "checkbox", "name": "exactmatch", ...uglyChecked(exactMatch)}, null, null, "change", (e) => { saveToLocalStorage("rendertest_exactmatch", e.target.checked); - buildDataSlowMode(table, images, data, tableSort, filter.value, exactmatch_input.checked); + buildDataSlowMode(table, images, TEST_RESULTS, tableSort, filter.value, exactmatch_input.checked); }), Text("exact match") ]) @@ -586,7 +594,7 @@ window.onload = () => { Tag("div", {"class": "content"}, null, [ Tag("h1", null, "Rendertest report"), Tag("h2", null, "Summary"), - Tag("div", {"id": "summary"}, null, [...buildSummary(data)]), + Tag("div", {"id": "summary"}, null, buildSummary(TEST_RESULTS)), Tag("h2", null, "Images of things that are not perfect ", [ Tag("span", {"class": "emoji"}, "โคต") ]), @@ -595,7 +603,7 @@ window.onload = () => { ]) document.body.appendChild(gridContainer); - buildData(table, images, data, tableSort, filter_value, exactmatch_input.checked); + buildData(table, images, TEST_RESULTS, tableSort, filter_value, exactmatch_input.checked); // TODO: remove this if (sidebarPos === "right") { diff --git a/render/rendertest.py b/render/rendertest.py index af1d5a6..49fd93e 100755 --- a/render/rendertest.py +++ b/render/rendertest.py @@ -2,12 +2,14 @@ import argparse import concurrent.futures +from datetime import datetime import json import os import pathlib import re import shutil import subprocess +import time ROOT = os.path.dirname(os.path.abspath(__file__)) imagecompare = f'{ROOT}/imagecompare' @@ -15,11 +17,11 @@ WORKDIR = f'{ROOT}/work' REPORT_ROOT = f'{ROOT}' # FIXME should be workdir? def get_imagemagick_convert_cmd() -> str: - if shutil.which("magick"): - return "magick" - if shutil.which("convert"): - return "convert" - raise RuntimeError("No ImageMagick binary found (tried: magick, convert)") + if shutil.which("magick"): + return "magick" + if shutil.which("convert"): + return "convert" + raise RuntimeError("No ImageMagick binary found (tried: magick, convert)") convert = get_imagemagick_convert_cmd() # TODO rename to __ @@ -66,10 +68,28 @@ parser.add_argument('--tests', '-t', type=test_list, default=saves, help='Run on # TODO how to check that the dir is valid? presence of xash3d executable and valve dir? parser.add_argument('--xash-dir', '-x', type=str, default=os.getcwd(), help='Path to xash3d-fwgs installation directory') +parser.add_argument('--xash-revision', type=str, default='unknown', help='xash engine repo revision, commit_id<-dirty>') +parser.add_argument('--pbr-revision', type=str, default='unknown', help='PBR textures repo revision, commit_id<-dirty>') +parser.add_argument('--tests-revision', type=str, default='unknown', help='HLRTest (this) repo revision, commit_id<-dirty>') + # TODO parse commands in type=.. function parser.add_argument('command', type=str, default=None, help='Action to perform') args = parser.parse_args() + +metadata = {} +metadata['time'] = datetime.now().astimezone().isoformat() +metadata['revisions'] = { + 'xash3d': args.xash_revision, + 'pbr': args.pbr_revision, + 'tests': args.tests_revision, +} +# TODO: +# - OS/kernel version +# - GPU model +# - Vulkan version +# - GPU driver version + def make_script(file, tests: [str]): header = '''sv_cheats 1 developer 0 @@ -132,6 +152,7 @@ def render(): env = os.environ.copy() env['LD_LIBRARY_PATH'] = f'{args.xash_dir}' + start_time_sec = time.perf_counter() with open(f'{WORKDIR}/xash-stdout.log', 'wb') as stdout, open(f'{WORKDIR}/xash-stderr.log', 'wb') as stderr: result = subprocess.run([f'{args.xash_dir}/xash3d', '-ref', 'vk', '-nowriteconfig', '-nosound', '-log', @@ -139,6 +160,8 @@ def render(): '-width', '1280', '-height', '800', '+exec', 'rendertest.script'], env=env, check=True, stdout=stdout, stderr=stderr) + end_time_sec = time.perf_counter() + metadata['phase_render_time_sec'] = end_time_sec - start_time_sec def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str): result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True) @@ -165,6 +188,7 @@ def compare(): diffs = [] command_png() print(f'Comparing...') + start_time_sec = time.perf_counter() with concurrent.futures.ThreadPoolExecutor() as executor: for test in args.tests: for channel, _ in channels.items(): @@ -175,11 +199,19 @@ def compare(): diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff)) - results = [diff.result() for diff in diffs] + tests = [diff.result() for diff in diffs] + end_time_sec = time.perf_counter() + metadata['phase_compare_time_sec'] = end_time_sec - start_time_sec + + results = { + 'metadata': metadata, + 'tests': tests, + } + json.dump(results, open(f'{WORKDIR}/test_results.json', 'w')) jsons = json.dumps(results) - with open(f'{WORKDIR}/data.js', 'w') as js: - js.write(f'"use strict";\nconst data = {jsons};\n') + with open(f'{WORKDIR}/results.js', 'w') as js: + js.write(f'"use strict";\nconst TEST_RESULTS = {jsons};\n') def command_compare(): compile() @@ -215,6 +247,8 @@ def command_render(): copy_assets() render() +# TODO make one long sequence of functions, and then call enabled ones, passing state via files + # TODO dict match args.command: case 'run': diff --git a/render/report.html b/render/report.html index 19e35f8..3edd2ea 100644 --- a/render/report.html +++ b/render/report.html @@ -8,7 +8,7 @@ - +