pass extra metadata (repo refs, timestamps) to test results

container/build-and-test.sh

@@ -20,7 +20,12 @@ build() {
 
 rendertest() {
 	pushd /build/HLRTest/render
-	WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py --xash-dir /opt/hl run
+	WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py \
+		--xash-dir /opt/hl \
+		--xash-revision "${XASH3D_REVISION}" \
+		--pbr-revision "${PBR_REVISION}" \
+		--tests-revision "${HLRTEST_REVISION}" \
+		run
 	popd
 }
 

container/run.sh

@@ -3,7 +3,7 @@ set -eux
 
 NAME=xash-builder
 
-HLRTEST_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+HLRTEST_REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 
 source .env
 
@@ -20,10 +20,37 @@ build() {
 	build-image
 }
 
+git_short() {
+	local dir="${1:-.}"
+	local hash dirty_flag
+
+	# Get short commit hash
+	hash=$(cd "$dir" && git rev-parse --short HEAD 2>/dev/null)
+
+	if [[ -z "$hash" ]]; then
+		echo "Not a git repository" >&2
+		return 1
+	fi
+
+	# Check for uncommitted changes (staged or unstaged)
+	dirty_flag=$(cd "$dir" && git status --porcelain --untracked-files=no 2>/dev/null)
+
+	if [[ -n "$dirty_flag" ]]; then
+		echo "${hash}-dirty"
+	else
+		echo "$hash"
+	fi
+}
+
 render-test() {
-	OUTPUT="${HLRTEST_PATH}/render/work"
+	local OUTPUT="${HLRTEST_REPO_DIR}/render/work"
-	# Make sure the image ubuntu user can write to the output
+
-	chmod -R o+rw "${OUTPUT}"
+	local HLRTEST_REVISION=$(git_short "${HLRTEST_REPO_DIR}")
+	local XASH3D_REVISION=$(git_short "${XASH3D_RT_REPO_DIR}")
+	local PBR_REVISION=$(git_short "${HALFLIFE_PBR_REPO_DIR}")
+
+	# Make sure the image's ubuntu (uid=1000) user can write to the output
+	#chmod -R o+rw "${OUTPUT}"
 
 	podman run -it --rm \
 		--name ${NAME} \
@@ -34,14 +61,20 @@ render-test() {
 		--read-only \
 		--tmpfs /tmp \
 		\
+		-e HLRTEST_REVISION="${HLRTEST_REVISION}" \
+		-e XASH3D_REVISION="${XASH3D_REVISION}" \
+		-e PBR_REVISION="${PBR_REVISION}" \
+		\
 		-v /dev/dri/renderD128:/dev/dri/renderD128:ro \
 		\
 		-v /opt/hl \
+		-v /home/ubuntu \
 		-v ${HALFLIFE_PBR_REPO_DIR}/valve/pbr:/opt/hl/valve/pbr:ro \
+		-v ${HALFLIFE_PBR_REPO_DIR}/valve/bluenoise:/opt/hl/valve/bluenoise:ro \
 		\
 		-v ${XASH3D_RT_REPO_DIR}:/build/xash3d-fwgs:O \
 		\
-		-v ${HLRTEST_PATH}:/build/HLRTest:ro \
+		-v ${HLRTEST_REPO_DIR}:/build/HLRTest:ro \
 		-v ${OUTPUT}:/build/HLRTest/render/work \
 		\
 		ubuntu-xash-builder:latest \

render/index.js

@@ -114,7 +114,7 @@ function isPassed(value) {
 	];
 }
 
-function makeTable(fields, data, attrs_func) {
+function makeTable(fields, tests, attrs_func) {
 	const table = Tag('table', null, null, [
 		Tag('tr', null, null, (() => {
 			let tds = [];
@@ -126,8 +126,8 @@ function makeTable(fields, data, attrs_func) {
 	]);
 
 
-	for (const di in data) {
+	for (const di in tests) {
-		let d = data[di];
+		let d = tests[di];
 		const attrs = attrs_func ? attrs_func(d) : null;
 		table.appendChild(Tag('tr', attrs, null, (() => {
 			let ret = [];
@@ -159,7 +159,7 @@ function makeTable(fields, data, attrs_func) {
 	return table;
 }
 
-function buildTestResultsTable(data) {
+function buildTestResultsTable(tests) {
 	const fields = [
 		{label: 'Test', field: 'test', tags_func: sectionLink},
 		{label: 'Channel', field: 'channel', tags_func: channelWithEmoji},
@@ -167,27 +167,37 @@ function buildTestResultsTable(data) {
 		{label: 'Passed', field: 'fail', tags_func: isPassed},
 	];
 
-	return [makeTable(fields, data, rowAttribs)];
+	return [makeTable(fields, tests, rowAttribs)];
 }
 
 // Filter out all success
-function filterData(data) {
+function filterTests(tests) {
-	return data
+	return tests
-	.filter((d) => { return d.diff_pct !== 0; })
+		.filter((t) => { return t.diff_pct !== 0; })
-	.sort((l, r) => { 
+		.sort((l, r) => { 
-		return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM
+			return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM
-	});
+		});
 }
 
-function buildSummary(data) {
+function buildSummary(test_results) {
-	const total = data.length;
+	const tests = test_results.tests
-	const fails = data.filter((d) => { return !!d.fail; }).length;
+	const total = tests.length;
+	const fails = tests.filter((d) => { return !!d.fail; }).length;
 	const fails_pct = fails * 100.0 / total;
-	return emojiToTag(`🧪 Tests 💥: ${fails}/${total}🏁 (⚠${fails_pct.toFixed(3)}%)`);
+	const meta = test_results.metadata;
+	return [
+		Tag('ul', null, null, [
+			Tag('li', null, `Test date: ${meta.time}`),
+			// TODO add links to repos at specified revisions
+			Tag('li', null, `Xash3D-FWGS revision: ${meta.revisions.xash3d}`),
+			Tag('li', null, `Half-Life-PBR revision: ${meta.revisions.pbr}`),
+			Tag('li', null, `HLRTest revision: ${meta.revisions.tests}`),
+			Tag('li', null, `Rendering took: ${meta.phase_render_time_sec.toFixed(1)} seconds`)
+		])].concat(emojiToTag(`🧪 Tests 💥: ${fails}/${total}🏁 (⚠${fails_pct.toFixed(3)}%)`));
 }
 
-function buildTestResultImages(data) {
+function buildTestResultImages(tests) {
-	return data.flatMap((d) => {
+	return tests.flatMap((d) => {
 		//return Tag('details', {id: makeId(d)}, null, [
 		return Tag('div', {id: makeId(d), "class": "meta-block"}, null, [
 			//Tag('summary', null, `${d.test}/${d.channel} δ=${d.diff_pct}`),
@@ -207,10 +217,8 @@ function buildTestResultImages(data) {
 
 
 
-function buildData(table, images, data, sort, filter, exact_match) {
+function buildData(table, images, test_results, sort, filter, exact_match) {
-	if (sort) {
+	let tests = sort ? filterTests(test_results.tests) : test_results.tests
-		data = filterData(data);
-	}
 	if (filter) {
 		let test = "", channel = "";
 		const args = filter.split(" ");
@@ -221,7 +229,7 @@ function buildData(table, images, data, sort, filter, exact_match) {
 			test = filter;
 			channel = filter;
 		}
-		data = data.filter((d) => {
+		tests = tests.filter((d) => {
 			const test_result = exact_match ? d.test === test : d.test.includes(test);
 			const channel_result = exact_match ? d.channel === channel : d.channel.includes(channel);
 			if (args.length > 1) {
@@ -232,8 +240,8 @@ function buildData(table, images, data, sort, filter, exact_match) {
 		});
 	}
 
-	table.replaceChildren(...buildTestResultsTable(data));
+	table.replaceChildren(...buildTestResultsTable(tests));
-	images.replaceChildren(...buildTestResultImages(data));
+	images.replaceChildren(...buildTestResultImages(tests));
 
 	for (let block of images.querySelectorAll(".image-container")) {
 		block.addEventListener("click", function() {
@@ -255,8 +263,8 @@ function buildData(table, images, data, sort, filter, exact_match) {
 
 	saveToLocalStorage("rendertest_tablesort", sort);
 }
-const buildDataSlowMode = debounce((table, images, data, sort, filter, exact_match) => {
+const buildDataSlowMode = debounce((table, images, test_results, sort, filter, exact_match) => {
-	buildData(table, images, data, sort, filter, exact_match);
+	buildData(table, images, test_results, sort, filter, exact_match);
 }, ()=>{}, 250);
 
 
@@ -550,7 +558,7 @@ window.onload = () => {
 					]), 
 					Tag("label", null, null, [
 						Tag("input", {"type": "radio", "name": "sort_table", "value": "yes", ...uglyChecked(tableSort)}, null, null, "input", (e) => {
-							buildData(table, images, data, true, filter.value, exactmatch_input.checked);
+							buildData(table, images, TEST_RESULTS, true, filter.value, exactmatch_input.checked);
 							// TODO: rewrite use reactive programming
 							tableSort = true;
 						}),
@@ -558,7 +566,7 @@ window.onload = () => {
 					]), 
 					Tag("label", null, null, [
 						Tag("input", {"type": "radio", "name": "sort_table", "value": "no", ...uglyChecked(!tableSort)}, null, null, "input", (e) => {
-							buildData(table, images, data, false, filter.value, exactmatch_input.checked);
+							buildData(table, images, TEST_RESULTS, false, filter.value, exactmatch_input.checked);
 							tableSort = false;
 						}),
 						Text(" no")
@@ -570,12 +578,12 @@ window.onload = () => {
 				Tag("label", {"class": "filter sticky"}, "Filter", [
 					filter = Tag("input", {"type": "input", "title": "Hotkey: ALT+F", "name": "filter", "value": filter_value}, null, null, "input", (e) => {
 						saveToLocalStorage("rendertest_filter", e.target.value);
-						buildDataSlowMode(table, images, data, tableSort, e.target.value, exactmatch_input.checked);
+						buildDataSlowMode(table, images, TEST_RESULTS, tableSort, e.target.value, exactmatch_input.checked);
 					}),
 					Tag("label", null, null, [
 						exactmatch_input = Tag("input", {"type": "checkbox", "name": "exactmatch", ...uglyChecked(exactMatch)}, null, null, "change", (e) => {
 							saveToLocalStorage("rendertest_exactmatch", e.target.checked);
-							buildDataSlowMode(table, images, data, tableSort, filter.value, exactmatch_input.checked);
+							buildDataSlowMode(table, images, TEST_RESULTS, tableSort, filter.value, exactmatch_input.checked);
 						}),
 						Text("exact match")
 					])
@@ -586,7 +594,7 @@ window.onload = () => {
 		Tag("div", {"class": "content"}, null, [
 			Tag("h1", null, "Rendertest report"),
 			Tag("h2", null, "Summary"),
-			Tag("div", {"id": "summary"}, null, [...buildSummary(data)]),
+			Tag("div", {"id": "summary"}, null, buildSummary(TEST_RESULTS)),
 			Tag("h2", null, "Images of things that are not perfect ", [
 				Tag("span", {"class": "emoji"}, "⤵")
 			]),
@@ -595,7 +603,7 @@ window.onload = () => {
 	])
 	document.body.appendChild(gridContainer);
 
-	buildData(table, images, data, tableSort, filter_value, exactmatch_input.checked);
+	buildData(table, images, TEST_RESULTS, tableSort, filter_value, exactmatch_input.checked);
 
 	// TODO: remove this
 	if (sidebarPos === "right") {

render/rendertest.py

@@ -2,12 +2,14 @@
 
 import argparse
 import concurrent.futures
+from datetime import datetime
 import json
 import os
 import pathlib
 import re
 import shutil
 import subprocess
+import time
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
 imagecompare = f'{ROOT}/imagecompare'
@@ -15,11 +17,11 @@ WORKDIR = f'{ROOT}/work'
 REPORT_ROOT = f'{ROOT}' # FIXME should be workdir?
 
 def get_imagemagick_convert_cmd() -> str:
-    if shutil.which("magick"):
+	if shutil.which("magick"):
-        return "magick"
+		return "magick"
-    if shutil.which("convert"):
+	if shutil.which("convert"):
-        return "convert"
+		return "convert"
-    raise RuntimeError("No ImageMagick binary found (tried: magick, convert)")
+	raise RuntimeError("No ImageMagick binary found (tried: magick, convert)")
 convert = get_imagemagick_convert_cmd()
 
 # TODO rename to <map>_<description>_<issue(opt)>
@@ -66,10 +68,28 @@ parser.add_argument('--tests', '-t', type=test_list, default=saves, help='Run on
 # TODO how to check that the dir is valid? presence of xash3d executable and valve dir?
 parser.add_argument('--xash-dir', '-x', type=str, default=os.getcwd(), help='Path to xash3d-fwgs installation directory')
 
+parser.add_argument('--xash-revision', type=str, default='unknown', help='xash engine repo revision, commit_id<-dirty>') 
+parser.add_argument('--pbr-revision', type=str, default='unknown', help='PBR textures repo revision, commit_id<-dirty>') 
+parser.add_argument('--tests-revision', type=str, default='unknown', help='HLRTest (this) repo revision, commit_id<-dirty>') 
+
 # TODO parse commands in type=.. function
 parser.add_argument('command', type=str, default=None, help='Action to perform')
 args = parser.parse_args()
 
+
+metadata = {}
+metadata['time'] = datetime.now().astimezone().isoformat()
+metadata['revisions'] = {
+	'xash3d': args.xash_revision,
+	'pbr': args.pbr_revision,
+	'tests': args.tests_revision,
+}
+# TODO:
+# - OS/kernel version
+# - GPU model
+# - Vulkan version
+# - GPU driver version
+
 def make_script(file, tests: [str]):
 	header = '''sv_cheats 1
 developer 0
@@ -132,6 +152,7 @@ def render():
 	env = os.environ.copy()
 	env['LD_LIBRARY_PATH'] = f'{args.xash_dir}'
 
+	start_time_sec = time.perf_counter()
 	with open(f'{WORKDIR}/xash-stdout.log', 'wb') as stdout, open(f'{WORKDIR}/xash-stderr.log', 'wb') as stderr:
 		result = subprocess.run([f'{args.xash_dir}/xash3d', '-ref', 'vk',
 			'-nowriteconfig', '-nosound', '-log',
@@ -139,6 +160,8 @@ def render():
 			'-width', '1280', '-height', '800',
 			'+exec', 'rendertest.script'],
 			env=env, check=True, stdout=stdout, stderr=stderr)
+	end_time_sec = time.perf_counter()
+	metadata['phase_render_time_sec'] = end_time_sec - start_time_sec
 
 def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
 	result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
@@ -165,6 +188,7 @@ def compare():
 	diffs = []
 	command_png()
 	print(f'Comparing...')
+	start_time_sec = time.perf_counter()
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
 			for channel, _ in channels.items():
@@ -175,11 +199,19 @@ def compare():
 
 				diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff))
 
-	results = [diff.result() for diff in diffs]
+	tests = [diff.result() for diff in diffs]
+	end_time_sec = time.perf_counter()
+	metadata['phase_compare_time_sec'] = end_time_sec - start_time_sec
+
+	results = {
+		'metadata': metadata,
+		'tests': tests,
+	}
+
 	json.dump(results, open(f'{WORKDIR}/test_results.json', 'w'))
 	jsons = json.dumps(results)
-	with open(f'{WORKDIR}/data.js', 'w') as js:
+	with open(f'{WORKDIR}/results.js', 'w') as js:
-		js.write(f'"use strict";\nconst data = {jsons};\n')
+		js.write(f'"use strict";\nconst TEST_RESULTS = {jsons};\n')
 
 def command_compare():
 	compile()
@@ -215,6 +247,8 @@ def command_render():
 	copy_assets()
 	render()
 
+# TODO make one long sequence of functions, and then call enabled ones, passing state via files
+
 # TODO dict
 match args.command:
 	case 'run':

render/report.html

@@ -8,7 +8,7 @@
 		<link rel="stylesheet alternate" type="text/css" href="theme-light.css" disabled="true" data-theme="theme-light" title="☀️ Light theme">
 		<meta name="viewport" content="width=device-width, initial-scale=1">
 		<script src="utils.js"></script>
-		<script src="work/data.js"></script>
+		<script src="work/results.js"></script>
 		<script src="index.js"></script>
 	</head>
 	<body>