render: add html report generation
This commit is contained in:
Ivan Avdeev
parent
bb5c04efd8
commit
c1bd3d7619
@ -56,18 +56,22 @@ static void imageFree(image_t *img) {
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
if (argc < 4) {
|
||||
fprintf(stderr, "Usage: %s infile1 infile2 out_diff.png\n", argv[0]);
|
||||
fprintf(stderr, "Usage: %s infile1 infile2 out_diff\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* const filename_gold = argv[1];
|
||||
const char* const filename_test = argv[2];
|
||||
const char* const filename_diff = argv[3];
|
||||
|
||||
const uint64_t start_ns = now();
|
||||
|
||||
image_t a, b;
|
||||
|
||||
if (!imageLoad(&a, argv[1]))
|
||||
if (!imageLoad(&a, filename_gold))
|
||||
return 1;
|
||||
|
||||
if (!imageLoad(&b, argv[2]))
|
||||
if (!imageLoad(&b, filename_test))
|
||||
return 1;
|
||||
|
||||
if (a.w != b.w || a.h != b.h || a.comp != b.comp) {
|
||||
@ -130,13 +134,30 @@ int main(int argc, char *argv[]) {
|
||||
const float diff_pct_threshold = 1.f;
|
||||
const float diff_pct = diff_sum * 100.f / total;
|
||||
const int over = diff_pct_threshold < diff_pct;
|
||||
// TODO count of pixels with difference over the threshold
|
||||
|
||||
fprintf(stderr, "%s\"%s\" vs \"%s\": %d (%.03f%%)\033[0m\n",
|
||||
over ? "\033[31mFAIL" : (diff_sum == 0 ? "\033[32m" : ""),
|
||||
argv[1], argv[2], diff_sum, diff_pct
|
||||
);
|
||||
|
||||
if (!imageSave(&diff, argv[3]))
|
||||
// X(name, format, expression)
|
||||
#define LIST_PROPS(X) \
|
||||
X(diff_total, "%d, ", diff_sum) \
|
||||
X(diff_pct, "%.03f, ", diff_pct) \
|
||||
X(fail, "%d", over) \
|
||||
|
||||
fprintf(stdout, "{"
|
||||
#define X(name, format, expr) "\"" #name "\": " format
|
||||
LIST_PROPS(X)
|
||||
#undef X
|
||||
"}\n"
|
||||
#define X(name, format, expr) , (expr)
|
||||
LIST_PROPS(X)
|
||||
#undef X
|
||||
);
|
||||
|
||||
if (!imageSave(&diff, filename_diff))
|
||||
return 1;
|
||||
|
||||
const uint64_t end_ns = now();
|
||||
|
||||
110
render/index.js
Normal file
110
render/index.js
Normal file
@ -0,0 +1,110 @@
|
||||
"use strict";
|
||||
|
||||
const fieldFuncDefault = (value) => {
|
||||
return [Text(value)];
|
||||
};
|
||||
|
||||
function makeId(d) {
|
||||
return `${d.test}/${d.channel}`;
|
||||
}
|
||||
|
||||
function rowAttribs(d) {
|
||||
return d.fail ? {style: `background-color: rgb(128, 0, 0)`} : null;
|
||||
}
|
||||
|
||||
function sectionLink(value, d) {
|
||||
const id = makeId(d);
|
||||
const a = Tag('a', {href: '#'+id}, value);
|
||||
a.addEventListener('click', () => { $(id).setAttribute('open', true); });
|
||||
return [a];
|
||||
}
|
||||
|
||||
function makeTable(fields, data, attrs_func) {
|
||||
const table = Tag('table', null, null, [
|
||||
Tag('tr', null, null, (() => {
|
||||
let tds = [];
|
||||
for (const f of fields) {
|
||||
tds.push(Tag('td', null, null, [Text(f.label)]));
|
||||
}
|
||||
return tds;
|
||||
})()),
|
||||
]);
|
||||
|
||||
for (const di in data) {
|
||||
let d = data[di];
|
||||
const attrs = attrs_func ? attrs_func(d) : null;
|
||||
table.appendChild(Tag('tr', attrs, null, (() => {
|
||||
let ret = [];
|
||||
for (const f of fields) {
|
||||
const value = d[f.field];
|
||||
if (value === undefined) {
|
||||
ret.push(Tag('td'));
|
||||
continue;
|
||||
}
|
||||
|
||||
const field_func = f.tags_func ? f.tags_func : fieldFuncDefault;
|
||||
const tags = field_func(value, d);
|
||||
let attrs = null;
|
||||
|
||||
// if (f.good && f.bad) {
|
||||
// const rating = linearStep(value, f.good, f.bad);
|
||||
// const c = {
|
||||
// r: 224 + 31 * clamp(2 - rating * 2, 0, 1),
|
||||
// g: 224 + 31 * clamp(rating * 2, 0, 1),
|
||||
// b: 224,
|
||||
// };
|
||||
// attrs = {style: `background-color: rgb(${c.r}, ${c.g}, ${c.b})`};
|
||||
// }
|
||||
ret.push(Tag('td', attrs, null, tags));
|
||||
}
|
||||
return ret;
|
||||
})()));
|
||||
}
|
||||
|
||||
return table;
|
||||
}
|
||||
|
||||
function buildTestResultsTable(data) {
|
||||
const fields = [
|
||||
{label: 'Test', field: 'test', tags_func: sectionLink},
|
||||
{label: 'Channel', field: 'channel', tags_func: sectionLink},
|
||||
{label: 'Diff, %', field: 'diff_pct'},
|
||||
{label: 'Failed', field: 'fail'},
|
||||
];
|
||||
|
||||
return [makeTable(fields, data, rowAttribs)];
|
||||
}
|
||||
|
||||
// Filter out all success
|
||||
function filterData(data) {
|
||||
return data
|
||||
.filter((d) => {return d.diff_pct != 0;})
|
||||
.sort((l, r) => {return l.diff_pct < r.diff_pct;});
|
||||
}
|
||||
|
||||
function buildSummary(data) {
|
||||
const total = data.length;
|
||||
const fails = data.filter((d) => {return !!d.fail;}).length;
|
||||
const fails_pct = fails * 100.0 / total;
|
||||
return [Text(`Tests failed: ${fails} / ${total} (${fails_pct.toFixed(3)}%)`)];
|
||||
}
|
||||
|
||||
function buildTestResultImages(data) {
|
||||
return data.flatMap((d) => {
|
||||
return Tag('details', {id: makeId(d)}, null, [
|
||||
Tag('summary', null, `${d.test}/${d.channel} diff=${d.diff_pct}`),
|
||||
Tag('img', {src: d.image_diff}),
|
||||
Tag('img', {src: d.image_flip}),
|
||||
]);
|
||||
});
|
||||
}
|
||||
|
||||
window.onload = () => {
|
||||
$('summary').replaceChildren(...buildSummary(data));
|
||||
|
||||
const filtered_data = filterData(data);
|
||||
$('fail_table').replaceChildren(...buildTestResultsTable(filtered_data));
|
||||
$('fail_images').replaceChildren(...buildTestResultImages(filtered_data));
|
||||
}
|
||||
|
||||
|
||||
@ -2,12 +2,16 @@
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
imagecompare = f'{ROOT}/imagecompare'
|
||||
WORKDIR = f'{ROOT}/work'
|
||||
REPORT_ROOT = f'{ROOT}' # FIXME should be workdir?
|
||||
|
||||
# TODO load all saves from the save/ dir
|
||||
# TODO rename to <map>_<description>_<issue(opt)>
|
||||
@ -21,7 +25,7 @@ saves = [
|
||||
'c1a3_fan_material_669',
|
||||
]
|
||||
|
||||
displays = {
|
||||
channels = {
|
||||
'full': '',
|
||||
'basecolor': 'basecolor',
|
||||
'emissive': 'emissive',
|
||||
@ -77,8 +81,8 @@ rt_debug_fixed_random_seed 31337
|
||||
file.write(f'wait 4; echo DONE WAIT4; playersonly; wait 11\n')
|
||||
# for i in range(13):
|
||||
# file.write(f'echo FRAME {i+4}; wait 1;\n')
|
||||
for name, display in displays.items():
|
||||
file.write(f'rt_debug_display_only "{display}"; screenshot {screenshot_base}{test}_{name}.tga; wait 1\n')
|
||||
for channel, display in channels.items():
|
||||
file.write(f'rt_debug_display_only "{display}"; screenshot {screenshot_base}{test}_{channel}.tga; wait 1\n')
|
||||
file.write('\n')
|
||||
|
||||
file.write('quit\n')
|
||||
@ -92,13 +96,13 @@ def mkdir_p(path: str):
|
||||
|
||||
def compile():
|
||||
subprocess.run(['make', 'imagecompare'], cwd=ROOT, check=True)
|
||||
with open(f'{args.xash_dir}/rendertest.script', 'w') as script:
|
||||
make_script(script, args.tests)
|
||||
|
||||
def copy_assets():
|
||||
print('Copying assets')
|
||||
shutil.copytree(src=f'{ROOT}/maps', dst=f'{args.xash_dir}/valve/maps/', dirs_exist_ok=True)
|
||||
shutil.copytree(src=f'{ROOT}/save', dst=f'{args.xash_dir}/valve/save/', dirs_exist_ok=True)
|
||||
with open(f'{args.xash_dir}/rendertest.script', 'w') as script:
|
||||
make_script(script, args.tests)
|
||||
|
||||
def render():
|
||||
print('Running xash3d')
|
||||
@ -112,9 +116,9 @@ def render():
|
||||
'+exec', 'rendertest.script'],
|
||||
env=env, check=True)
|
||||
|
||||
def compare_one(imagecompare: str, image_base: str, image_gold: str, image_test: str, image_diff: str):
|
||||
compare = subprocess.run([imagecompare, image_gold, image_test, image_diff])
|
||||
match compare.returncode:
|
||||
def compare_one(test: str, channel: str, image_base: str, image_gold: str, image_test: str, image_diff: str, image_flip: str):
|
||||
result = subprocess.run([imagecompare, image_gold, image_test, image_diff], text=True, capture_output=True)
|
||||
match result.returncode:
|
||||
case 0:
|
||||
pass
|
||||
case 1:
|
||||
@ -122,38 +126,53 @@ def compare_one(imagecompare: str, image_base: str, image_gold: str, image_test:
|
||||
case 2:
|
||||
print(f'ERROR: {image_base} differ by more than threshold')
|
||||
case _:
|
||||
raise Exception(f'Unexpected imagecompare return code {compare.returncode}')
|
||||
raise Exception(f'Unexpected imagecompare return code {result.returncode}')
|
||||
ret = json.loads(result.stdout)
|
||||
ret['test'] = test
|
||||
ret['channel'] = channel
|
||||
ret['image_gold'] = os.path.relpath(image_gold, REPORT_ROOT)
|
||||
ret['image_test'] = os.path.relpath(image_test, REPORT_ROOT)
|
||||
ret['image_diff'] = os.path.relpath(image_diff, REPORT_ROOT)
|
||||
ret['image_flip'] = os.path.relpath(image_flip, REPORT_ROOT)
|
||||
return ret
|
||||
|
||||
def compare():
|
||||
screenshot_base = f'{args.xash_dir}/valve/rendertest'
|
||||
imagecompare = f'{ROOT}/imagecompare'
|
||||
diffs = []
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
for test in args.tests:
|
||||
for name, display in displays.items():
|
||||
image_base = f'{test}_{name}'
|
||||
for channel, _ in channels.items():
|
||||
image_base = f'{test}_{channel}'
|
||||
image_test = f'{screenshot_base}/{image_base}.tga'
|
||||
image_gold = f'{ROOT}/gold/{image_base}.png'
|
||||
image_diff = f'{ROOT}/work/{image_base}_diff.tga'
|
||||
image_diff = f'{WORKDIR}/{image_base}_diff.png'
|
||||
image_flip = f'{WORKDIR}/{image_base}_flip.gif'
|
||||
|
||||
diffs.append(executor.submit(compare_one, imagecompare, image_base, image_gold, image_test, image_diff))
|
||||
diffs.append(executor.submit(compare_one, test, channel, image_base, image_gold, image_test, image_diff, image_flip))
|
||||
|
||||
executor.submit(subprocess.run, ['convert',
|
||||
'(', image_gold, '-bordercolor', 'gold', '-border', '2x2', '-gravity', 'SouthWest', '-font', 'Impact', '-pointsize', '24', '-fill', 'gold', '-stroke', 'black', '-annotate', '0', 'GOLD', ')',
|
||||
'(', image_test, '-bordercolor', 'white', '-border', '2x2', '-fill', 'white', '-annotate', '0', 'TEST', ')',
|
||||
'-loop', '0', '-set', 'delay', '25', f'{ROOT}/work/{image_base}_flip.gif'], check=True)
|
||||
'-loop', '0', '-set', 'delay', '25', image_flip], check=True)
|
||||
|
||||
results = [diff.result() for diff in diffs]
|
||||
# json.dump(results, open(f'{WORKDIR}/data.json', 'w'))
|
||||
jsons = json.dumps(results)
|
||||
with open(f'{WORKDIR}/data.js', 'w') as js:
|
||||
js.write(f'"use strict";\nconst data = {jsons};\n')
|
||||
|
||||
def command_compare():
|
||||
compile()
|
||||
compare()
|
||||
|
||||
def command_png():
|
||||
screenshot_base = f'{args.xash_dir}/valve/rendertest'
|
||||
new_gold_base = f'{ROOT}/work/gold'
|
||||
new_gold_base = f'{WORKDIR}/gold'
|
||||
mkdir_p(new_gold_base)
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
for test in args.tests:
|
||||
for name, display in displays.items():
|
||||
image_base = f'{test}_{name}'
|
||||
for channel, _ in channels.items():
|
||||
image_base = f'{test}_{channel}'
|
||||
image_test = f'{screenshot_base}/{image_base}.tga'
|
||||
image_new_gold = f'{new_gold_base}/{image_base}.png'
|
||||
|
||||
@ -172,6 +191,7 @@ def command_render():
|
||||
copy_assets()
|
||||
render()
|
||||
|
||||
# TODO dict
|
||||
match args.command:
|
||||
case 'run':
|
||||
command_run()
|
||||
|
||||
24
render/report.html
Normal file
24
render/report.html
Normal file
@ -0,0 +1,24 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Rendertest report</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<script src="utils.js"></script>
|
||||
<script src="work/data.js"></script>
|
||||
<script src="index.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<!-- TODO have a date and build number or whatever -->
|
||||
<h1>Rendertest report</h1>
|
||||
<h2>Summary</h2>
|
||||
<div id="summary"></div>
|
||||
|
||||
<h2>List of things that are not perfect</h2>
|
||||
<div id="fail_table"></div>
|
||||
|
||||
<h2>Images of things that are not perfect</h2>
|
||||
<div id="fail_images"></div>
|
||||
</body>
|
||||
</html>
|
||||
71
render/style.css
Normal file
71
render/style.css
Normal file
@ -0,0 +1,71 @@
|
||||
html, body {
|
||||
box-sizing: border-box;
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
*, *:before, *:after {
|
||||
box-sizing: inherit;
|
||||
}
|
||||
|
||||
#main-wrapper {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 100%;
|
||||
}
|
||||
|
||||
#main-content {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
footer {
|
||||
text-align: center;
|
||||
color: #888;
|
||||
margin: .5em;
|
||||
}
|
||||
|
||||
.autocomplete {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.autocomplete-items {
|
||||
z-index: 99;
|
||||
position: absolute;
|
||||
top: 100%;
|
||||
left: 0;
|
||||
right: 0;
|
||||
}
|
||||
|
||||
.autocomplete-item {
|
||||
background-color: #fefefe;
|
||||
cursor: pointer;
|
||||
border: 1px solid #eee;
|
||||
border-top: none;
|
||||
}
|
||||
|
||||
.autocomplete-item:hover {
|
||||
background-color: #e9e9e9;
|
||||
}
|
||||
|
||||
.item-inactive {
|
||||
padding: 5px;
|
||||
}
|
||||
|
||||
.item-inactive .group {
|
||||
color: #777;
|
||||
}
|
||||
|
||||
.item-active {
|
||||
padding: 5px;
|
||||
background-color: DodgerBlue !important;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.item-active .group {
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
table, tr, td, th {
|
||||
border: 1px solid #888;
|
||||
}
|
||||
62
render/utils.js
Normal file
62
render/utils.js
Normal file
@ -0,0 +1,62 @@
|
||||
"use strict";
|
||||
|
||||
function $(name) { return document.getElementById(name); }
|
||||
|
||||
function Tag(name, attrs, body, children) {
|
||||
let elem = document.createElement(name);
|
||||
if (body) {
|
||||
elem.innerHTML = body;
|
||||
}
|
||||
for (let k in attrs) {
|
||||
elem.setAttribute(k, attrs[k]);
|
||||
}
|
||||
for (let i in children) {
|
||||
elem.appendChild(children[i]);
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
|
||||
function Text(text) {
|
||||
return document.createTextNode(text);
|
||||
}
|
||||
|
||||
function debounce(func, cancelFunc, timeout = 500) {
|
||||
let timer;
|
||||
return (...args) => {
|
||||
cancelFunc();
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(() => { func.apply(this, args); }, timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function sendRequest(method, path, query, body, funcDone, funcError) {
|
||||
let req = new XMLHttpRequest();
|
||||
if (query) {
|
||||
let params = new URLSearchParams(query);
|
||||
path = path + "?" + params.toString();
|
||||
}
|
||||
req.open(method, path, true);
|
||||
req.onreadystatechange = function () {
|
||||
if (req.readyState == XMLHttpRequest.DONE) {
|
||||
let status = req.status;
|
||||
if (status === 0 || (status >= 200 && status < 400)) {
|
||||
if (funcDone)
|
||||
funcDone(req.responseText, status);
|
||||
} else {
|
||||
if (funcError)
|
||||
funcError(req.responseText, status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (body) {
|
||||
req.setRequestHeader("Content-Type", "application/json");
|
||||
console.log("SENDING", body)
|
||||
req.send(JSON.stringify(body));
|
||||
} else {
|
||||
req.send();
|
||||
}
|
||||
|
||||
return req;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user