1
0
mirror of https://github.com/emilk/egui.git synced 2026-08-29 04:40:03 -04:00

Report failing pixels by threshold when a kittest snapshot fails (#8360)

When an image snapshot fails, you get the number of pixels differing by
more than the `threshold` you happened to configure — which doesn't tell
you what threshold *would* have passed. So picking
`SnapshotOptions::threshold` / `failed_pixel_count_threshold` is trial
and error, one CI round-trip per guess.

This measures the failing pixel count at a sweep of thresholds (new
public `THRESHOLD_SWEEP`) and includes it in `SnapshotError::Diff`:

```
'sweep_demo' Image did not match snapshot. Diff: 293, …/sweep_demo.diff.png.
  Failing pixels by threshold: 0.0: 1522, 0.1: 1522, 0.2: 293, 0.4: 293, 0.6: 293, 1.0: 293, …
  Run `UPDATE_SNAPSHOTS=1 cargo test --all-features` to update the snapshots.
```

The sweep only runs for snapshots that already failed, so passing tests
are unaffected.

Breaking: `SnapshotError::Diff` gained a `failing_pixels_by_threshold`
field.

* [x] I have followed the instructions in the PR template

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Emil Ernerfeldt
2026-08-03 01:08:09 -07:00
committed by GitHub
parent 967aa1137a
commit c676d939ca

View File

@@ -214,6 +214,14 @@ pub enum SnapshotError {
/// Path where the diff image was saved /// Path where the diff image was saved
diff_path: PathBuf, diff_path: PathBuf,
/// How many pixels would have failed at other per-pixel thresholds.
///
/// Measured at [`THRESHOLD_SWEEP`], lowest threshold first.
/// Use this to pick a [`SnapshotOptions::threshold`] and a
/// [`SnapshotOptions::failed_pixel_count_threshold`] from measurements,
/// instead of by trial and error.
failing_pixels_by_threshold: Vec<(f32, i32)>,
}, },
/// Error opening the existing snapshot (it probably doesn't exist, check the /// Error opening the existing snapshot (it probably doesn't exist, check the
@@ -264,14 +272,24 @@ impl Display for SnapshotError {
name, name,
diff, diff,
diff_path, diff_path,
failing_pixels_by_threshold,
} => { } => {
let diff_path = let diff_path =
std::path::absolute(diff_path).unwrap_or_else(|_| diff_path.clone()); std::path::absolute(diff_path).unwrap_or_else(|_| diff_path.clone());
write!( write!(
f, f,
"'{name}' Image did not match snapshot. Diff: {diff}, {}. {HOW_TO_UPDATE_SCREENSHOTS}", "'{name}' Image did not match snapshot. Diff: {diff}, {}.",
diff_path.display() diff_path.display()
) )?;
if !failing_pixels_by_threshold.is_empty() {
let sweep = failing_pixels_by_threshold
.iter()
.map(|(threshold, count)| format!("{threshold:.1}: {count}"))
.collect::<Vec<_>>()
.join(", ");
write!(f, "\n Failing pixels by threshold: {sweep}")?;
}
write!(f, "\n {HOW_TO_UPDATE_SCREENSHOTS}")
} }
Self::OpenSnapshot { path, err } => { Self::OpenSnapshot { path, err } => {
let path = std::path::absolute(path).unwrap_or_else(|_| path.clone()); let path = std::path::absolute(path).unwrap_or_else(|_| path.clone());
@@ -380,6 +398,37 @@ pub fn try_image_snapshot_options(
try_image_snapshot_options_impl(new, name.into(), options) try_image_snapshot_options_impl(new, name.into(), options)
} }
/// The per-pixel thresholds that a failing snapshot is measured against,
/// to help you pick a [`SnapshotOptions::threshold`].
///
/// Same unit as [`SnapshotOptions::threshold`].
pub const THRESHOLD_SWEEP: &[f32] = &[0.0, 0.1, 0.2, 0.4, 0.6, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0];
/// How many pixels differ by more than each of [`THRESHOLD_SWEEP`]?
///
/// Only called for failing snapshots, so the extra comparisons don't slow down passing tests.
fn failing_pixels_by_threshold(
previous: &image::RgbaImage,
new: &image::RgbaImage,
) -> Vec<(f32, i32)> {
THRESHOLD_SWEEP
.iter()
.map(|&threshold| {
let num_wrong_pixels = dify::diff::get_results(
previous.clone(),
new.clone(),
threshold,
true,
None,
&None,
&None,
)
.map_or(0, |(num_wrong_pixels, _diff_image)| num_wrong_pixels);
(threshold, num_wrong_pixels)
})
.collect()
}
fn try_image_snapshot_options_impl( fn try_image_snapshot_options_impl(
new: &image::RgbaImage, new: &image::RgbaImage,
name: String, name: String,
@@ -481,8 +530,15 @@ fn try_image_snapshot_options_impl(
*threshold *threshold
}; };
let result = let result = dify::diff::get_results(
dify::diff::get_results(previous, new.clone(), threshold, true, None, &None, &None); previous.clone(),
new.clone(),
threshold,
true,
None,
&None,
&None,
);
let Some((num_wrong_pixels, diff_image)) = result else { let Some((num_wrong_pixels, diff_image)) = result else {
return Ok(()); // Difference below threshold return Ok(()); // Difference below threshold
@@ -510,6 +566,7 @@ fn try_image_snapshot_options_impl(
name, name,
diff: num_wrong_pixels, diff: num_wrong_pixels,
diff_path, diff_path,
failing_pixels_by_threshold: failing_pixels_by_threshold(&previous, new),
}) })
} }
} }