render: add html report generation

render/imagecompare.c

@@ -56,18 +56,22 @@ static void imageFree(image_t *img) {
 
 int main(int argc, char *argv[]) {
 	if (argc < 4) {
-		fprintf(stderr, "Usage: %s infile1 infile2 out_diff.png\n", argv[0]);
+		fprintf(stderr, "Usage: %s infile1 infile2 out_diff\n", argv[0]);
 		return 1;
 	}
 
+	const char* const filename_gold = argv[1];
+	const char* const filename_test = argv[2];
+	const char* const filename_diff = argv[3];
+
 	const uint64_t start_ns = now();
 
 	image_t a, b;
 
-	if (!imageLoad(&a, argv[1]))
+	if (!imageLoad(&a, filename_gold))
 		return 1;
 
-	if (!imageLoad(&b, argv[2]))
+	if (!imageLoad(&b, filename_test))
 		return 1;
 
 	if (a.w != b.w || a.h != b.h || a.comp != b.comp) {
@@ -130,13 +134,30 @@ int main(int argc, char *argv[]) {
 	const float diff_pct_threshold = 1.f;
 	const float diff_pct = diff_sum * 100.f / total;
 	const int over = diff_pct_threshold < diff_pct;
+	// TODO count of pixels with difference over the threshold
 
 	fprintf(stderr, "%s\"%s\" vs \"%s\": %d (%.03f%%)\033[0m\n",
 		over ? "\033[31mFAIL" : (diff_sum == 0 ? "\033[32m" : ""),
 		argv[1], argv[2], diff_sum, diff_pct
 		);
 
-	if (!imageSave(&diff, argv[3]))
+// X(name, format, expression)
+#define LIST_PROPS(X) \
+	X(diff_total, "%d, ", diff_sum) \
+	X(diff_pct, "%.03f, ", diff_pct) \
+	X(fail, "%d", over) \
+
+	fprintf(stdout, "{"
+#define X(name, format, expr) "\"" #name "\": " format
+		LIST_PROPS(X)
+#undef X
+		"}\n"
+#define X(name, format, expr) , (expr)
+		LIST_PROPS(X)
+#undef X
+	);
+
+	if (!imageSave(&diff, filename_diff))
 		return 1;
 
 	const uint64_t end_ns = now();

render/index.js

@@ -0,0 +1,110 @@
+"use strict";
+
+const fieldFuncDefault = (value) => {
+	return [Text(value)];
+};
+
+function makeId(d) {
+	return `${d.test}/${d.channel}`;
+}
+
+function rowAttribs(d) {
+	return d.fail ? {style: `background-color: rgb(128, 0, 0)`} : null;
+}
+
+function sectionLink(value, d) {
+	const id = makeId(d);
+	const a = Tag('a', {href: '#'+id}, value);
+	a.addEventListener('click', () => { $(id).setAttribute('open', true); });
+	return [a];
+}
+
+function makeTable(fields, data, attrs_func) {
+	const table = Tag('table', null, null, [
+		Tag('tr', null, null, (() => {
+			let tds = [];
+			for (const f of fields) {
+				tds.push(Tag('td', null, null, [Text(f.label)]));
+			}
+			return tds;
+		})()),
+	]);
+
+	for (const di in data) {
+		let d = data[di];
+		const attrs = attrs_func ? attrs_func(d) : null;
+		table.appendChild(Tag('tr', attrs, null, (() => {
+			let ret = [];
+			for (const f of fields) {
+				const value = d[f.field];
+				if (value === undefined) {
+					ret.push(Tag('td'));
+					continue;
+				}
+
+				const field_func = f.tags_func ? f.tags_func : fieldFuncDefault;
+				const tags = field_func(value, d);
+				let attrs = null;
+
+				// if (f.good && f.bad) {
+				// 	const rating = linearStep(value, f.good, f.bad);
+				// 	const c = {
+				// 		r: 224 + 31 * clamp(2 - rating * 2, 0, 1),
+				// 		g: 224 + 31 * clamp(rating * 2, 0, 1),
+				// 		b: 224,
+				// 	};
+				// 	attrs = {style: `background-color: rgb(${c.r}, ${c.g}, ${c.b})`};
+				// }
+				ret.push(Tag('td', attrs, null, tags));
+			}
+			return ret;
+		})()));
+	}
+
+	return table;
+}
+
+function buildTestResultsTable(data) {
+	const fields = [
+		{label: 'Test', field: 'test', tags_func: sectionLink},
+		{label: 'Channel', field: 'channel', tags_func: sectionLink},
+		{label: 'Diff, %', field: 'diff_pct'},
+		{label: 'Failed', field: 'fail'},
+	];
+
+	return [makeTable(fields, data, rowAttribs)];
+}
+
+// Filter out all success
+function filterData(data) {
+	return data
+		.filter((d) => {return d.diff_pct != 0;})
+		.sort((l, r) => {return l.diff_pct < r.diff_pct;});
+}
+
+function buildSummary(data) {
+	const total = data.length;
+	const fails = data.filter((d) => {return !!d.fail;}).length;
+	const fails_pct = fails * 100.0 / total;
+	return [Text(`Tests failed: ${fails} / ${total} (${fails_pct.toFixed(3)}%)`)];
+}
+
+function buildTestResultImages(data) {
+	return data.flatMap((d) => {
+		return Tag('details', {id: makeId(d)}, null, [
+			Tag('summary', null, `${d.test}/${d.channel} diff=${d.diff_pct}`),
+			Tag('img', {src: d.image_diff}),
+			Tag('img', {src: d.image_flip}),
+		]);
+	});
+}
+
+window.onload = () => {
+	$('summary').replaceChildren(...buildSummary(data));
+
+	const filtered_data = filterData(data);
+	$('fail_table').replaceChildren(...buildTestResultsTable(filtered_data));
+	$('fail_images').replaceChildren(...buildTestResultImages(filtered_data));
+}
+
+

render/rendertest.py

@@ -2,12 +2,16 @@
 
 import argparse
 import concurrent.futures
+import json
 import os
 import pathlib
 import shutil
 import subprocess
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
+imagecompare = f'{ROOT}/imagecompare'
+WORKDIR = f'{ROOT}/work'
+REPORT_ROOT = f'{ROOT}' # FIXME should be workdir?
 
 # TODO load all saves from the save/ dir
 # TODO rename to <map>_<description>_<issue(opt)>
@@ -21,7 +25,7 @@ saves = [
 	'c1a3_fan_material_669',
 ]
 
-displays = {
+channels = {
 	'full': '',
 	'basecolor': 'basecolor',
 	'emissive': 'emissive',
@@ -77,8 +81,8 @@ rt_debug_fixed_random_seed 31337
 		file.write(f'wait 4; echo DONE WAIT4; playersonly; wait 11\n')
 		# for i in range(13):
 		# 	file.write(f'echo FRAME {i+4}; wait 1;\n')
-		for name, display in displays.items():
+		for channel, display in channels.items():
-			file.write(f'rt_debug_display_only "{display}"; screenshot {screenshot_base}{test}_{name}.tga; wait 1\n')
+			file.write(f'rt_debug_display_only "{display}"; screenshot {screenshot_base}{test}_{channel}.tga; wait 1\n')
 		file.write('\n')
 
 	file.write('quit\n')
@@ -92,13 +96,13 @@ def mkdir_p(path: str):
 
 def compile():
 	subprocess.run(['make', 'imagecompare'], cwd=ROOT, check=True)
-	with open(f'{args.xash_dir}/rendertest.script', 'w') as script:
-		make_script(script, args.tests)
 
 def copy_assets():
 	print('Copying assets')
 	shutil.copytree(src=f'{ROOT}/maps', dst=f'{args.xash_dir}/valve/maps/', dirs_exist_ok=True)
 	shutil.copytree(src=f'{ROOT}/save', dst=f'{args.xash_dir}/valve/save/', dirs_exist_ok=True)
+	with open(f'{args.xash_dir}/rendertest.script', 'w') as script:
+		make_script(script, args.tests)
 
 def render():
 	print('Running xash3d')
@@ -112,9 +116,9 @@ def render():
 		'+exec', 'rendertest.script'],
 		env=env, check=True)
 
-def compare_one(imagecompare: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
+def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str, image_flip: str):
-	compare = subprocess.run([imagecompare, image_gold, image_test, image_diff])
+	result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
-	match compare.returncode:
+	match result.returncode:
 		case 0:
 			pass
 		case 1:
@@ -122,38 +126,53 @@ def compare_one(imagecompare: str, image_base: str, image_gold: str, image_test:
 		case 2:
 			print(f'ERROR: {image_base} differ by more than threshold')
 		case _:
-			raise Exception(f'Unexpected imagecompare return code {compare.returncode}')
+			raise Exception(f'Unexpected imagecompare return code {result.returncode}')
+	ret = json.loads(result.stdout)
+	ret['test'] = test
+	ret['channel'] = channel
+	ret['image_gold'] = os.path.relpath(image_gold, REPORT_ROOT)
+	ret['image_test'] = os.path.relpath(image_test, REPORT_ROOT)
+	ret['image_diff'] = os.path.relpath(image_diff, REPORT_ROOT)
+	ret['image_flip'] = os.path.relpath(image_flip, REPORT_ROOT)
+	return ret
 
 def compare():
 	screenshot_base = f'{args.xash_dir}/valve/rendertest'
-	imagecompare = f'{ROOT}/imagecompare'
 	diffs = []
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
-			for name, display in displays.items():
+			for channel, _ in channels.items():
-				image_base = f'{test}_{name}'
+				image_base = f'{test}_{channel}'
 				image_test = f'{screenshot_base}/{image_base}.tga'
 				image_gold = f'{ROOT}/gold/{image_base}.png'
-				image_diff = f'{ROOT}/work/{image_base}_diff.tga'
+				image_diff = f'{WORKDIR}/{image_base}_diff.png'
+				image_flip = f'{WORKDIR}/{image_base}_flip.gif'
 
-				diffs.append(executor.submit(compare_one, imagecompare, image_base, image_gold, image_test, image_diff))
+				diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff, image_flip))
 
 				executor.submit(subprocess.run, ['convert',
 					'(', image_gold, '-bordercolor', 'gold', '-border', '2x2', '-gravity', 'SouthWest', '-font', 'Impact', '-pointsize', '24', '-fill', 'gold', '-stroke', 'black', '-annotate', '0', 'GOLD', ')',
 					'(', image_test, '-bordercolor', 'white', '-border', '2x2', '-fill', 'white', '-annotate', '0', 'TEST', ')',
-					'-loop', '0', '-set', 'delay', '25', f'{ROOT}/work/{image_base}_flip.gif'], check=True)
+					'-loop', '0', '-set', 'delay', '25', image_flip], check=True)
+
+	results = [diff.result() for diff in diffs]
+	# json.dump(results, open(f'{WORKDIR}/data.json', 'w'))
+	jsons = json.dumps(results)
+	with open(f'{WORKDIR}/data.js', 'w') as js:
+		js.write(f'"use strict";\nconst data = {jsons};\n')
 
 def command_compare():
+	compile()
 	compare()
 
 def command_png():
 	screenshot_base = f'{args.xash_dir}/valve/rendertest'
-	new_gold_base = f'{ROOT}/work/gold'
+	new_gold_base = f'{WORKDIR}/gold'
 	mkdir_p(new_gold_base)
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
-			for name, display in displays.items():
+			for channel, _ in channels.items():
-				image_base = f'{test}_{name}'
+				image_base = f'{test}_{channel}'
 				image_test = f'{screenshot_base}/{image_base}.tga'
 				image_new_gold = f'{new_gold_base}/{image_base}.png'
 
@@ -172,6 +191,7 @@ def command_render():
 	copy_assets()
 	render()
 
+# TODO dict
 match args.command:
 	case 'run':
 		command_run()

render/report.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="UTF-8">
+		<title>Rendertest report</title>
+		<link rel="stylesheet" href="style.css">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<script src="utils.js"></script>
+		<script src="work/data.js"></script>
+		<script src="index.js"></script>
+	</head>
+	<body>
+		<!-- TODO have a date and build number or whatever -->
+		<h1>Rendertest report</h1>
+		<h2>Summary</h2>
+		<div id="summary"></div>
+
+		<h2>List of things that are not perfect</h2>
+		<div id="fail_table"></div>
+
+		<h2>Images of things that are not perfect</h2>
+		<div id="fail_images"></div>
+	</body>
+</html>

render/style.css

@@ -0,0 +1,71 @@
+html, body {
+  box-sizing: border-box;
+	height: 100%;
+	margin: 0;
+	padding: 0;
+}
+*, *:before, *:after {
+  box-sizing: inherit;
+}
+
+#main-wrapper {
+	display: flex;
+	flex-direction: column;
+	min-height: 100%;
+}
+
+#main-content {
+	flex: 1;
+}
+
+footer {
+	text-align: center;
+	color: #888;
+	margin: .5em;
+}
+
+.autocomplete {
+	position: relative;
+	display: inline-block;
+}
+
+.autocomplete-items {
+	z-index: 99;
+	position: absolute;
+	top: 100%;
+	left: 0;
+	right: 0;
+}
+
+.autocomplete-item {
+	background-color: #fefefe;
+	cursor: pointer;
+	border: 1px solid #eee;
+	border-top: none;
+}
+
+.autocomplete-item:hover {
+	background-color: #e9e9e9;
+}
+
+.item-inactive {
+	padding: 5px;
+}
+
+.item-inactive .group {
+	color: #777;
+}
+
+.item-active {
+	padding: 5px;
+	background-color: DodgerBlue !important;
+	color: #fff;
+}
+
+.item-active .group {
+	color: #ccc;
+}
+
+table, tr, td, th {
+	border: 1px solid #888;
+}

render/utils.js

@@ -0,0 +1,62 @@
+"use strict";
+
+function $(name) { return document.getElementById(name); }
+
+function Tag(name, attrs, body, children) {
+	let elem = document.createElement(name);
+	if (body) {
+		elem.innerHTML = body;
+	}
+	for (let k in attrs) {
+		elem.setAttribute(k, attrs[k]);
+	}
+	for (let i in children) {
+		elem.appendChild(children[i]);
+	}
+	return elem;
+}
+
+function Text(text) {
+	return document.createTextNode(text);
+}
+
+function debounce(func, cancelFunc, timeout = 500) {
+	let timer;
+	return (...args) => {
+		cancelFunc();
+		clearTimeout(timer);
+		timer = setTimeout(() => { func.apply(this, args); }, timeout);
+	}
+}
+
+function sendRequest(method, path, query, body, funcDone, funcError) {
+	let req = new XMLHttpRequest();
+	if (query) {
+		let params = new URLSearchParams(query);
+		path = path + "?" + params.toString();
+	}
+	req.open(method, path, true);
+	req.onreadystatechange = function () {
+		if (req.readyState == XMLHttpRequest.DONE) {
+			let status = req.status;
+			if (status === 0 || (status >= 200 && status < 400)) {
+				if (funcDone)
+					funcDone(req.responseText, status);
+			} else {
+				if (funcError)
+					funcError(req.responseText, status);
+			}
+		}
+	}
+
+	if (body) {
+		req.setRequestHeader("Content-Type", "application/json");
+		console.log("SENDING", body)
+		req.send(JSON.stringify(body));
+	} else {
+		req.send();
+	}
+
+	return req;
+}
+