update c2a5-skybox for asphalt material

lunarg no longer updates ubuntu packages, and those are stale

.gitignore

@@ -1,3 +1,4 @@
 render/work
 imagecompare
 rendertest.script
+container/Half-Life

container/Containerfile

@@ -0,0 +1,67 @@
+FROM ubuntu-hlsdk
+
+# Avoid interactive prompts during package installation
+ENV DEBIAN_FRONTEND=noninteractive
+ENV VULKAN_SDK_VERSION=1.4.341.1
+
+USER root
+
+RUN apt-get update && apt-get full-upgrade -y && apt-get autoremove -y
+
+# Install system dependencies and development packages
+RUN apt-get install --no-install-recommends --no-install-suggests -y \
+    libsdl2-dev \
+    libfreetype-dev \
+    \
+    # FFmpeg development packages
+    libavcodec-dev \
+    libavformat-dev \
+    libavutil-dev \
+    libavfilter-dev \
+    libavdevice-dev \
+    libswscale-dev \
+    libswresample-dev \
+    libpostproc-dev \
+    ffmpeg \
+    \
+    # Running render tests
+    weston \
+    libgl1-mesa-dri \
+    mesa-vulkan-drivers \
+    libvulkan1 \
+    imagemagick
+
+# Install Vulkan SDK runtime dependencies only
+#RUN apt-get update && apt-get install -y \
+#   libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev
+
+# Install Vulkan SDK
+RUN mkdir -p /opt/VulkanSDK && wget -qO- \
+    https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.xz | \
+  tar -xJ -C /opt/VulkanSDK/
+
+ENV VULKAN_SDK=/opt/VulkanSDK/${VULKAN_SDK_VERSION}/x86_64
+ENV PATH=$VULKAN_SDK/bin:$PATH
+ENV LD_LIBRARY_PATH="$VULKAN_SDK/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+ENV VK_ADD_LAYER_PATH="$VULKAN_SDK/share/vulkan/explicit_layer.d${VK_ADD_LAYER_PATH:+:$VK_ADD_LAYER_PATH}"
+ENV PKG_CONFIG_PATH="$VULKAN_SDK/share/pkgconfig:$VULKAN_SDK/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}"
+
+# Common dev utils
+RUN apt-get install --no-install-recommends --no-install-suggests -y \
+    ripgrep \
+    less \
+    fd-find
+
+# Remove extra cache after all the installations
+RUN rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /build
+COPY build-and-test.sh /build/
+
+# Switch to non-root user
+USER ubuntu
+
+# Default command
+WORKDIR /build
+CMD ["/build/build-and-test.sh"]

container/Containerfile.hlsdk

@@ -0,0 +1,48 @@
+FROM ubuntu:24.04
+
+# This cannot reference external dirs, unfortunately
+ARG HALFLIFE_STEAM_FILES=./Half-Life
+
+# Copy HL data files
+RUN mkdir -p /opt/hl && chown ubuntu:ubuntu /opt/hl
+COPY --chown=ubuntu:ubuntu ${HALFLIFE_STEAM_FILES}/valve /opt/hl/valve
+COPY --chown=ubuntu:ubuntu ${HALFLIFE_STEAM_FILES}/valve_hd /opt/hl/valve_hd
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install generic build dependencies
+RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \
+    \
+    # Build essential utilities
+    ca-certificates \
+    wget \
+    build-essential \
+    pkg-config \
+    python3 \
+    git
+
+# Build FWGS hlsdk game libs
+ARG FWGS_HLSDK_REF=ae84bfc0c3598fcff605a7b3bb963abe8ec3e295
+RUN mkdir -p /build && chown ubuntu:ubuntu /build
+USER ubuntu
+RUN cd /build && \
+    git version && \
+    # git > 2.49: git clone --depth 1 --revision ${FWGS_HLSDK_REF} https://github.com/FWGS/hlsdk-portable && \
+    mkdir -p hlsdk-portable && \
+    cd hlsdk-portable && \
+    git init && \
+    git remote add origin https://github.com/FWGS/hlsdk-portable && \
+    git fetch --depth 1 origin ${FWGS_HLSDK_REF} && \
+    git checkout FETCH_HEAD && \
+    ./waf \
+        configure -8 -T release \
+        build \
+        install --destdir=/opt/hl
+
+# Cleanup build files
+RUN rm -rf /build/hlsdk-portable
+
+# At this point there's:
+# /opt/hl with
+#   ./valve
+#       {dlls, cl_dlls} with fwgs/hlsdk libs

container/build-and-test.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -eux
+
+echo DESTINATION_IS_WRITABLE > /opt/hl/testfile
+
+XDG_RUNTIME_DIR=/tmp weston \
+	--backend=headless \
+	--renderer=gl \
+	--width=1280 \
+	--height=800 \
+	--socket=wayland-headless &
+
+build() {
+	pushd /build/xash3d-fwgs
+	./waf configure -T release -8 \
+		build \
+		install --destdir=/opt/hl
+	popd
+}
+
+rendertest() {
+	pushd /build/HLRTest/render
+	WAYLAND_DISPLAY=/tmp/wayland-headless ./rendertest.py \
+		--xash-dir /opt/hl \
+		--xash-revision "${XASH3D_REVISION}" \
+		--pbr-revision "${PBR_REVISION}" \
+		--tests-revision "${HLRTEST_REVISION}" \
+		run
+	popd
+}
+
+time build
+time rendertest

container/env.example

@@ -0,0 +1,5 @@
+# Path to https://rtxash.omgwtf.ru/Half-Life-RTX/Half-Life-PBR repo clone (readonly)
+HALFLIFE_PBR_REPO_DIR=${HOME}/src/Half-Life-PBR
+
+# Path to https://github.com/w23/xash3d-fwgs/ clone (readonly)
+XASH3D_RT_REPO_DIR=${HOME}/src/xash3d-fwgs

container/run.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -eux
+
+# just podman things
+export DBUS_SESSION_BUS_ADDRESS=
+
+NAME=xash-builder
+
+HLRTEST_REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+
+source .env
+
+build-image-hlsdk() {
+	podman build -t ubuntu-hlsdk -f Containerfile.hlsdk
+}
+
+build-image() {
+	podman build -t ubuntu-xash-builder -f Containerfile
+}
+
+build() {
+	build-image-hlsdk
+	build-image
+}
+
+git_short() {
+	local dir="${1:-.}"
+	local hash dirty_flag
+
+	# Get short commit hash
+	hash=$(cd "$dir" && git rev-parse --short HEAD 2>/dev/null)
+
+	if [[ -z "$hash" ]]; then
+		echo "Not a git repository" >&2
+		return 1
+	fi
+
+	# Check for uncommitted changes (staged or unstaged)
+	dirty_flag=$(cd "$dir" && git status --porcelain --untracked-files=no 2>/dev/null)
+
+	if [[ -n "$dirty_flag" ]]; then
+		echo "${hash}-dirty"
+	else
+		echo "$hash"
+	fi
+}
+
+render-test() {
+	local OUTPUT="${HLRTEST_REPO_DIR}/render/work"
+	mkdir -p "${OUTPUT}"
+
+	local HLRTEST_REVISION=$(git_short "${HLRTEST_REPO_DIR}")
+	local XASH3D_REVISION=$(git_short "${XASH3D_RT_REPO_DIR}")
+	local PBR_REVISION=$(git_short "${HALFLIFE_PBR_REPO_DIR}")
+
+	# Make sure the image's ubuntu (uid=1000) user can write to the output
+	#chmod -R o+rw "${OUTPUT}"
+
+	podman run -it --rm \
+		--name ${NAME} \
+		--userns=keep-id:uid=1000,gid=1000 \
+		\
+		--cap-drop=ALL \
+		--security-opt=no-new-privileges \
+		--read-only \
+		--tmpfs /tmp \
+		\
+		-e HLRTEST_REVISION="${HLRTEST_REVISION}" \
+		-e XASH3D_REVISION="${XASH3D_REVISION}" \
+		-e PBR_REVISION="${PBR_REVISION}" \
+		\
+		-v /dev/dri/renderD128:/dev/dri/renderD128:ro \
+		-v /etc/localtime:/etc/localtime:ro \
+		\
+		-v /opt/hl \
+		-v /home/ubuntu \
+		-v ${HALFLIFE_PBR_REPO_DIR}/valve/pbr:/opt/hl/valve/pbr:ro \
+		-v ${HALFLIFE_PBR_REPO_DIR}/valve/bluenoise:/opt/hl/valve/bluenoise:ro \
+		\
+		-v ${XASH3D_RT_REPO_DIR}:/build/xash3d-fwgs:O \
+		\
+		-v ${HLRTEST_REPO_DIR}:/build/HLRTest:O \
+		-v ${OUTPUT}:/build/HLRTest/render/work \
+		\
+		ubuntu-xash-builder:latest \
+		"$@"
+}
+
+shell() {
+	podman exec -it ${NAME} /bin/bash
+}
+
+time "$@"

render/Makefile

@@ -1,2 +1,2 @@
 imagecompare: imagecompare.c stb_image.h stb_image_write.h Makefile
-	${CC} -O3 -ggdb3 -march=native -Wall -Werror -pedantic -lm -o imagecompare imagecompare.c
+	${CC} -O3 -ggdb3 -march=native -Wall -Werror -pedantic -o imagecompare imagecompare.c -lm

render/index.js

@@ -114,7 +114,7 @@ function isPassed(value) {
 	];
 }
 
-function makeTable(fields, data, attrs_func) {
+function makeTable(fields, tests, attrs_func) {
 	const table = Tag('table', null, null, [
 		Tag('tr', null, null, (() => {
 			let tds = [];
@@ -126,8 +126,8 @@ function makeTable(fields, data, attrs_func) {
 	]);
 
 
-	for (const di in data) {
-		let d = data[di];
+	for (const di in tests) {
+		let d = tests[di];
 		const attrs = attrs_func ? attrs_func(d) : null;
 		table.appendChild(Tag('tr', attrs, null, (() => {
 			let ret = [];
@@ -159,7 +159,7 @@ function makeTable(fields, data, attrs_func) {
 	return table;
 }
 
-function buildTestResultsTable(data) {
+function buildTestResultsTable(tests) {
 	const fields = [
 		{label: 'Test', field: 'test', tags_func: sectionLink},
 		{label: 'Channel', field: 'channel', tags_func: channelWithEmoji},
@@ -167,27 +167,37 @@ function buildTestResultsTable(data) {
 		{label: 'Passed', field: 'fail', tags_func: isPassed},
 	];
 
-	return [makeTable(fields, data, rowAttribs)];
+	return [makeTable(fields, tests, rowAttribs)];
 }
 
 // Filter out all success
-function filterData(data) {
-	return data
-	.filter((d) => { return d.diff_pct !== 0; })
+function filterTests(tests) {
+	return tests
+		.filter((t) => { return t.diff_pct !== 0; })
 		.sort((l, r) => { 
 			return l.diff_pct < r.diff_pct ? 1 : -1; // RTFM
 		});
 }
 
-function buildSummary(data) {
-	const total = data.length;
-	const fails = data.filter((d) => { return !!d.fail; }).length;
+function buildSummary(test_results) {
+	const tests = test_results.tests
+	const total = tests.length;
+	const fails = tests.filter((d) => { return !!d.fail; }).length;
 	const fails_pct = fails * 100.0 / total;
-	return emojiToTag(`🧪 Tests 💥: ${fails}/${total}🏁 (⚠${fails_pct.toFixed(3)}%)`);
+	const meta = test_results.metadata;
+	return [
+		Tag('ul', null, null, [
+			Tag('li', null, `Test date: ${meta.time}`),
+			// TODO add links to repos at specified revisions
+			Tag('li', null, `Xash3D-FWGS revision: ${meta.revisions.xash3d}`),
+			Tag('li', null, `Half-Life-PBR revision: ${meta.revisions.pbr}`),
+			Tag('li', null, `HLRTest revision: ${meta.revisions.tests}`),
+			Tag('li', null, `Rendering took: ${meta.phase_render_time_sec.toFixed(1)} seconds`)
+		])].concat(emojiToTag(`🧪 Tests 💥: ${fails}/${total}🏁 (⚠${fails_pct.toFixed(3)}%)`));
 }
 
-function buildTestResultImages(data) {
-	return data.flatMap((d) => {
+function buildTestResultImages(tests) {
+	return tests.flatMap((d) => {
 		//return Tag('details', {id: makeId(d)}, null, [
 		return Tag('div', {id: makeId(d), "class": "meta-block"}, null, [
 			//Tag('summary', null, `${d.test}/${d.channel} δ=${d.diff_pct}`),
@@ -207,10 +217,8 @@ function buildTestResultImages(data) {
 
 
 
-function buildData(table, images, data, sort, filter, exact_match) {
-	if (sort) {
-		data = filterData(data);
-	}
+function buildData(table, images, test_results, sort, filter, exact_match) {
+	let tests = sort ? filterTests(test_results.tests) : test_results.tests
 	if (filter) {
 		let test = "", channel = "";
 		const args = filter.split(" ");
@@ -221,7 +229,7 @@ function buildData(table, images, data, sort, filter, exact_match) {
 			test = filter;
 			channel = filter;
 		}
-		data = data.filter((d) => {
+		tests = tests.filter((d) => {
 			const test_result = exact_match ? d.test === test : d.test.includes(test);
 			const channel_result = exact_match ? d.channel === channel : d.channel.includes(channel);
 			if (args.length > 1) {
@@ -232,8 +240,8 @@ function buildData(table, images, data, sort, filter, exact_match) {
 		});
 	}
 
-	table.replaceChildren(...buildTestResultsTable(data));
-	images.replaceChildren(...buildTestResultImages(data));
+	table.replaceChildren(...buildTestResultsTable(tests));
+	images.replaceChildren(...buildTestResultImages(tests));
 
 	for (let block of images.querySelectorAll(".image-container")) {
 		block.addEventListener("click", function() {
@@ -255,8 +263,8 @@ function buildData(table, images, data, sort, filter, exact_match) {
 
 	saveToLocalStorage("rendertest_tablesort", sort);
 }
-const buildDataSlowMode = debounce((table, images, data, sort, filter, exact_match) => {
-	buildData(table, images, data, sort, filter, exact_match);
+const buildDataSlowMode = debounce((table, images, test_results, sort, filter, exact_match) => {
+	buildData(table, images, test_results, sort, filter, exact_match);
 }, ()=>{}, 250);
 
 
@@ -550,7 +558,7 @@ window.onload = () => {
 					]), 
 					Tag("label", null, null, [
 						Tag("input", {"type": "radio", "name": "sort_table", "value": "yes", ...uglyChecked(tableSort)}, null, null, "input", (e) => {
-							buildData(table, images, data, true, filter.value, exactmatch_input.checked);
+							buildData(table, images, TEST_RESULTS, true, filter.value, exactmatch_input.checked);
 							// TODO: rewrite use reactive programming
 							tableSort = true;
 						}),
@@ -558,7 +566,7 @@ window.onload = () => {
 					]), 
 					Tag("label", null, null, [
 						Tag("input", {"type": "radio", "name": "sort_table", "value": "no", ...uglyChecked(!tableSort)}, null, null, "input", (e) => {
-							buildData(table, images, data, false, filter.value, exactmatch_input.checked);
+							buildData(table, images, TEST_RESULTS, false, filter.value, exactmatch_input.checked);
 							tableSort = false;
 						}),
 						Text(" no")
@@ -570,12 +578,12 @@ window.onload = () => {
 				Tag("label", {"class": "filter sticky"}, "Filter", [
 					filter = Tag("input", {"type": "input", "title": "Hotkey: ALT+F", "name": "filter", "value": filter_value}, null, null, "input", (e) => {
 						saveToLocalStorage("rendertest_filter", e.target.value);
-						buildDataSlowMode(table, images, data, tableSort, e.target.value, exactmatch_input.checked);
+						buildDataSlowMode(table, images, TEST_RESULTS, tableSort, e.target.value, exactmatch_input.checked);
 					}),
 					Tag("label", null, null, [
 						exactmatch_input = Tag("input", {"type": "checkbox", "name": "exactmatch", ...uglyChecked(exactMatch)}, null, null, "change", (e) => {
 							saveToLocalStorage("rendertest_exactmatch", e.target.checked);
-							buildDataSlowMode(table, images, data, tableSort, filter.value, exactmatch_input.checked);
+							buildDataSlowMode(table, images, TEST_RESULTS, tableSort, filter.value, exactmatch_input.checked);
 						}),
 						Text("exact match")
 					])
@@ -586,7 +594,7 @@ window.onload = () => {
 		Tag("div", {"class": "content"}, null, [
 			Tag("h1", null, "Rendertest report"),
 			Tag("h2", null, "Summary"),
-			Tag("div", {"id": "summary"}, null, [...buildSummary(data)]),
+			Tag("div", {"id": "summary"}, null, buildSummary(TEST_RESULTS)),
 			Tag("h2", null, "Images of things that are not perfect ", [
 				Tag("span", {"class": "emoji"}, "⤵")
 			]),
@@ -595,7 +603,7 @@ window.onload = () => {
 	])
 	document.body.appendChild(gridContainer);
 
-	buildData(table, images, data, tableSort, filter_value, exactmatch_input.checked);
+	buildData(table, images, TEST_RESULTS, tableSort, filter_value, exactmatch_input.checked);
 
 	// TODO: remove this
 	if (sidebarPos === "right") {

render/rendertest.py

@@ -2,12 +2,14 @@
 
 import argparse
 import concurrent.futures
+from datetime import datetime
 import json
 import os
 import pathlib
 import re
 import shutil
 import subprocess
+import time
 
 ROOT = os.path.dirname(os.path.abspath(__file__))
 imagecompare = f'{ROOT}/imagecompare'
@@ -66,10 +68,28 @@ parser.add_argument('--tests', '-t', type=test_list, default=saves, help='Run on
 # TODO how to check that the dir is valid? presence of xash3d executable and valve dir?
 parser.add_argument('--xash-dir', '-x', type=str, default=os.getcwd(), help='Path to xash3d-fwgs installation directory')
 
+parser.add_argument('--xash-revision', type=str, default='unknown', help='xash engine repo revision, commit_id<-dirty>') 
+parser.add_argument('--pbr-revision', type=str, default='unknown', help='PBR textures repo revision, commit_id<-dirty>') 
+parser.add_argument('--tests-revision', type=str, default='unknown', help='HLRTest (this) repo revision, commit_id<-dirty>') 
+
 # TODO parse commands in type=.. function
 parser.add_argument('command', type=str, default=None, help='Action to perform')
 args = parser.parse_args()
 
+
+metadata = {}
+metadata['time'] = datetime.now().astimezone().isoformat()
+metadata['revisions'] = {
+	'xash3d': args.xash_revision,
+	'pbr': args.pbr_revision,
+	'tests': args.tests_revision,
+}
+# TODO:
+# - OS/kernel version
+# - GPU model
+# - Vulkan version
+# - GPU driver version
+
 def make_script(file, tests: [str]):
 	header = '''sv_cheats 1
 developer 0
@@ -130,8 +150,9 @@ def render():
 	print('Running xash3d...')
 	mkdir_p(f'{args.xash_dir}/valve/rendertest')
 	env = os.environ.copy()
-	env['LD_LIBRARY_PATH'] = f'{args.xash_dir}'
+	env['LD_LIBRARY_PATH'] = f'{args.xash_dir}:{env["LD_LIBRARY_PATH"]}'
 
+	start_time_sec = time.perf_counter()
 	with open(f'{WORKDIR}/xash-stdout.log', 'wb') as stdout, open(f'{WORKDIR}/xash-stderr.log', 'wb') as stderr:
 		result = subprocess.run([f'{args.xash_dir}/xash3d', '-ref', 'vk',
 			'-nowriteconfig', '-nosound', '-log',
@@ -139,6 +160,8 @@ def render():
 			'-width', '1280', '-height', '800',
 			'+exec', 'rendertest.script'],
 			env=env, check=True, stdout=stdout, stderr=stderr)
+	end_time_sec = time.perf_counter()
+	metadata['phase_render_time_sec'] = end_time_sec - start_time_sec
 
 def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
 	result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
@@ -165,6 +188,7 @@ def compare():
 	diffs = []
 	command_png()
 	print(f'Comparing...')
+	start_time_sec = time.perf_counter()
 	with concurrent.futures.ThreadPoolExecutor() as executor:
 		for test in args.tests:
 			for channel, _ in channels.items():
@@ -175,11 +199,19 @@ def compare():
 
 				diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff))
 
-	results = [diff.result() for diff in diffs]
+	tests = [diff.result() for diff in diffs]
+	end_time_sec = time.perf_counter()
+	metadata['phase_compare_time_sec'] = end_time_sec - start_time_sec
+
+	results = {
+		'metadata': metadata,
+		'tests': tests,
+	}
+
 	json.dump(results, open(f'{WORKDIR}/test_results.json', 'w'))
 	jsons = json.dumps(results)
-	with open(f'{WORKDIR}/data.js', 'w') as js:
-		js.write(f'"use strict";\nconst data = {jsons};\n')
+	with open(f'{WORKDIR}/results.js', 'w') as js:
+		js.write(f'"use strict";\nconst TEST_RESULTS = {jsons};\n')
 
 def command_compare():
 	compile()
@@ -215,6 +247,8 @@ def command_render():
 	copy_assets()
 	render()
 
+# TODO make one long sequence of functions, and then call enabled ones, passing state via files
+
 # TODO dict
 match args.command:
 	case 'run':

render/report.html

@@ -8,7 +8,7 @@
 		<link rel="stylesheet alternate" type="text/css" href="theme-light.css" disabled="true" data-theme="theme-light" title="☀️ Light theme">
 		<meta name="viewport" content="width=device-width, initial-scale=1">
 		<script src="utils.js"></script>
-		<script src="work/data.js"></script>
+		<script src="work/results.js"></script>
 		<script src="index.js"></script>
 	</head>
 	<body>