pass extra metadata (repo refs, timestamps) to test results

render/rendertest.py

@@ -2,12 +2,14 @@
 
 import argparse
 import concurrent.futures
+from datetime import datetime
 import json
 import os
 import pathlib
 import re
 import shutil
 import subprocess
+import time
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
 imagecompare = f'{ROOT}/imagecompare'
@@ -15,11 +17,11 @@ WORKDIR = f'{ROOT}/work'
 REPORT_ROOT = f'{ROOT}' # FIXME should be workdir?
 
 def get_imagemagick_convert_cmd() -> str:
-    if shutil.which("magick"):
-        return "magick"
-    if shutil.which("convert"):
-        return "convert"
-    raise RuntimeError("No ImageMagick binary found (tried: magick, convert)")
+	if shutil.which("magick"):
+		return "magick"
+	if shutil.which("convert"):
+		return "convert"
+	raise RuntimeError("No ImageMagick binary found (tried: magick, convert)")
 convert = get_imagemagick_convert_cmd()
 
 # TODO rename to <map>_<description>_<issue(opt)>
@@ -66,10 +68,28 @@ parser.add_argument('--tests', '-t', type=test_list, default=saves, help='Run on
 # TODO how to check that the dir is valid? presence of xash3d executable and valve dir?
 parser.add_argument('--xash-dir', '-x', type=str, default=os.getcwd(), help='Path to xash3d-fwgs installation directory')
 
+parser.add_argument('--xash-revision', type=str, default='unknown', help='xash engine repo revision, commit_id<-dirty>') 
+parser.add_argument('--pbr-revision', type=str, default='unknown', help='PBR textures repo revision, commit_id<-dirty>') 
+parser.add_argument('--tests-revision', type=str, default='unknown', help='HLRTest (this) repo revision, commit_id<-dirty>') 
+
 # TODO parse commands in type=.. function
 parser.add_argument('command', type=str, default=None, help='Action to perform')
 args = parser.parse_args()
 
+
+metadata = {}
+metadata['time'] = datetime.now().astimezone().isoformat()
+metadata['revisions'] = {
+	'xash3d': args.xash_revision,
+	'pbr': args.pbr_revision,
+	'tests': args.tests_revision,
+}
+# TODO:
+# - OS/kernel version
+# - GPU model
+# - Vulkan version
+# - GPU driver version
+
 def make_script(file, tests: [str]):
 	header = '''sv_cheats 1
 developer 0
@@ -132,6 +152,7 @@ def render():
 	env = os.environ.copy()
 	env['LD_LIBRARY_PATH'] = f'{args.xash_dir}'
 
+	start_time_sec = time.perf_counter()
 	with open(f'{WORKDIR}/xash-stdout.log', 'wb') as stdout, open(f'{WORKDIR}/xash-stderr.log', 'wb') as stderr:
 		result = subprocess.run([f'{args.xash_dir}/xash3d', '-ref', 'vk',
 			'-nowriteconfig', '-nosound', '-log',
@@ -139,6 +160,8 @@ def render():
 			'-width', '1280', '-height', '800',
 			'+exec', 'rendertest.script'],
 			env=env, check=True, stdout=stdout, stderr=stderr)
+	end_time_sec = time.perf_counter()
+	metadata['phase_render_time_sec'] = end_time_sec - start_time_sec
 
 def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
 	result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
@@ -165,6 +188,7 @@ def compare():
 	diffs = []
 	command_png()
 	print(f'Comparing...')
+	start_time_sec = time.perf_counter()
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
 			for channel, _ in channels.items():
@@ -175,11 +199,19 @@ def compare():
 
 				diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff))
 
-	results = [diff.result() for diff in diffs]
+	tests = [diff.result() for diff in diffs]
+	end_time_sec = time.perf_counter()
+	metadata['phase_compare_time_sec'] = end_time_sec - start_time_sec
+
+	results = {
+		'metadata': metadata,
+		'tests': tests,
+	}
+
 	json.dump(results, open(f'{WORKDIR}/test_results.json', 'w'))
 	jsons = json.dumps(results)
-	with open(f'{WORKDIR}/data.js', 'w') as js:
-		js.write(f'"use strict";\nconst data = {jsons};\n')
+	with open(f'{WORKDIR}/results.js', 'w') as js:
+		js.write(f'"use strict";\nconst TEST_RESULTS = {jsons};\n')
 
 def command_compare():
 	compile()
@@ -215,6 +247,8 @@ def command_render():
 	copy_assets()
 	render()
 
+# TODO make one long sequence of functions, and then call enabled ones, passing state via files
+
 # TODO dict
 match args.command:
 	case 'run':