render: add html report generation

render/rendertest.py

@@ -2,12 +2,16 @@
 
 import argparse
 import concurrent.futures
+import json
 import os
 import pathlib
 import shutil
 import subprocess
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
+imagecompare = f'{ROOT}/imagecompare'
+WORKDIR = f'{ROOT}/work'
+REPORT_ROOT = f'{ROOT}' # FIXME should be workdir?
 
 # TODO load all saves from the save/ dir
 # TODO rename to <map>_<description>_<issue(opt)>
@@ -21,7 +25,7 @@ saves = [
 	'c1a3_fan_material_669',
 ]
 
-displays = {
+channels = {
 	'full': '',
 	'basecolor': 'basecolor',
 	'emissive': 'emissive',
@@ -77,8 +81,8 @@ rt_debug_fixed_random_seed 31337
 		file.write(f'wait 4; echo DONE WAIT4; playersonly; wait 11\n')
 		# for i in range(13):
 		# 	file.write(f'echo FRAME {i+4}; wait 1;\n')
-		for name, display in displays.items():
-			file.write(f'rt_debug_display_only "{display}"; screenshot {screenshot_base}{test}_{name}.tga; wait 1\n')
+		for channel, display in channels.items():
+			file.write(f'rt_debug_display_only "{display}"; screenshot {screenshot_base}{test}_{channel}.tga; wait 1\n')
 		file.write('\n')
 
 	file.write('quit\n')
@@ -92,13 +96,13 @@ def mkdir_p(path: str):
 
 def compile():
 	subprocess.run(['make', 'imagecompare'], cwd=ROOT, check=True)
-	with open(f'{args.xash_dir}/rendertest.script', 'w') as script:
-		make_script(script, args.tests)
 
 def copy_assets():
 	print('Copying assets')
 	shutil.copytree(src=f'{ROOT}/maps', dst=f'{args.xash_dir}/valve/maps/', dirs_exist_ok=True)
 	shutil.copytree(src=f'{ROOT}/save', dst=f'{args.xash_dir}/valve/save/', dirs_exist_ok=True)
+	with open(f'{args.xash_dir}/rendertest.script', 'w') as script:
+		make_script(script, args.tests)
 
 def render():
 	print('Running xash3d')
@@ -112,9 +116,9 @@ def render():
 		'+exec', 'rendertest.script'],
 		env=env, check=True)
 
-def compare_one(imagecompare: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
-	compare = subprocess.run([imagecompare, image_gold, image_test, image_diff])
-	match compare.returncode:
+def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str, image_flip: str):
+	result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
+	match result.returncode:
 		case 0:
 			pass
 		case 1:
@@ -122,38 +126,53 @@ def compare_one(imagecompare: str, image_base: str, image_gold: str, image_test:
 		case 2:
 			print(f'ERROR: {image_base} differ by more than threshold')
 		case _:
-			raise Exception(f'Unexpected imagecompare return code {compare.returncode}')
+			raise Exception(f'Unexpected imagecompare return code {result.returncode}')
+	ret = json.loads(result.stdout)
+	ret['test'] = test
+	ret['channel'] = channel
+	ret['image_gold'] = os.path.relpath(image_gold, REPORT_ROOT)
+	ret['image_test'] = os.path.relpath(image_test, REPORT_ROOT)
+	ret['image_diff'] = os.path.relpath(image_diff, REPORT_ROOT)
+	ret['image_flip'] = os.path.relpath(image_flip, REPORT_ROOT)
+	return ret
 
 def compare():
 	screenshot_base = f'{args.xash_dir}/valve/rendertest'
-	imagecompare = f'{ROOT}/imagecompare'
 	diffs = []
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
-			for name, display in displays.items():
-				image_base = f'{test}_{name}'
+			for channel, _ in channels.items():
+				image_base = f'{test}_{channel}'
 				image_test = f'{screenshot_base}/{image_base}.tga'
 				image_gold = f'{ROOT}/gold/{image_base}.png'
-				image_diff = f'{ROOT}/work/{image_base}_diff.tga'
+				image_diff = f'{WORKDIR}/{image_base}_diff.png'
+				image_flip = f'{WORKDIR}/{image_base}_flip.gif'
 
-				diffs.append(executor.submit(compare_one, imagecompare, image_base, image_gold, image_test, image_diff))
+				diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff, image_flip))
 
 				executor.submit(subprocess.run, ['convert',
 					'(', image_gold, '-bordercolor', 'gold', '-border', '2x2', '-gravity', 'SouthWest', '-font', 'Impact', '-pointsize', '24', '-fill', 'gold', '-stroke', 'black', '-annotate', '0', 'GOLD', ')',
 					'(', image_test, '-bordercolor', 'white', '-border', '2x2', '-fill', 'white', '-annotate', '0', 'TEST', ')',
-					'-loop', '0', '-set', 'delay', '25', f'{ROOT}/work/{image_base}_flip.gif'], check=True)
+					'-loop', '0', '-set', 'delay', '25', image_flip], check=True)
+
+	results = [diff.result() for diff in diffs]
+	# json.dump(results, open(f'{WORKDIR}/data.json', 'w'))
+	jsons = json.dumps(results)
+	with open(f'{WORKDIR}/data.js', 'w') as js:
+		js.write(f'"use strict";\nconst data = {jsons};\n')
 
 def command_compare():
+	compile()
 	compare()
 
 def command_png():
 	screenshot_base = f'{args.xash_dir}/valve/rendertest'
-	new_gold_base = f'{ROOT}/work/gold'
+	new_gold_base = f'{WORKDIR}/gold'
 	mkdir_p(new_gold_base)
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
-			for name, display in displays.items():
-				image_base = f'{test}_{name}'
+			for channel, _ in channels.items():
+				image_base = f'{test}_{channel}'
 				image_test = f'{screenshot_base}/{image_base}.tga'
 				image_new_gold = f'{new_gold_base}/{image_base}.png'
 
@@ -172,6 +191,7 @@ def command_render():
 	copy_assets()
 	render()
 
+# TODO dict
 match args.command:
 	case 'run':
 		command_run()