Add kittest.toml config file (#7643)

* part of https://github.com/rerun-io/rerun/issues/10991

---------

Co-authored-by: lucasmerlin <8009393+lucasmerlin@users.noreply.github.com>

Cargo.lock

@@ -1457,7 +1457,9 @@ dependencies = [
  "kittest",
  "open",
  "pollster",
+ "serde",
  "tempfile",
+ "toml",
  "wgpu",
 ]
 
@@ -4036,6 +4038,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serial_windows"
 version = "0.1.0"
@@ -4491,11 +4502,26 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "toml"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "toml_edit"
@@ -4504,6 +4530,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
 dependencies = [
  "indexmap",
+ "serde",
+ "serde_spanned",
  "toml_datetime",
  "winnow",
 ]
@@ -5645,9 +5673,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.7.3"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1"
+checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
 dependencies = [
  "memchr",
 ]

Cargo.toml

@@ -131,6 +131,7 @@ syntect = { version = "5.3.0", default-features = false }
 tempfile = "3.23.0"
 thiserror = "2.0.17"
 tokio = "1.47.1"
+toml = "0.8"
 type-map = "0.5.1"
 unicode_names2 = { version = "2.0.0", default-features = false }
 unicode-segmentation = "1.12.0"

crates/egui_demo_lib/tests/snapshots/rendering_test/dpi_1.67.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b7d7e290b97a8042af3af3cd9ceb274950cf607dd7e9cd6c71d5a113d3b57a5
+oid sha256:3a3a9aa8383abfe4580be2cc9987f8123aeabf36bf8ec06029a9af64b9500ec9
-size 1206155
+size 1206157

crates/egui_kittest/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "egui_kittest"
 version.workspace = true
-authors = ["Lucas Meurer <lucasmeurer96@gmail.com>", "Emil Ernerfeldt <emil.ernerfeldt@gmail.com>"]
+authors = ["Lucas Meurer <hi@lucasmerlin.me>", "Emil Ernerfeldt <emil.ernerfeldt@gmail.com>"]
 description = "Testing library for egui based on kittest and AccessKit"
 edition.workspace = true
 rust-version.workspace = true
@@ -34,9 +34,11 @@ x11 = ["eframe?/x11"]
 
 
 [dependencies]
-kittest.workspace = true
 egui.workspace = true
 eframe = { workspace = true, optional = true }
+kittest.workspace = true
+serde.workspace = true
+toml.workspace = true
 
 # wgpu dependencies
 egui-wgpu = { workspace = true, optional = true }

crates/egui_kittest/README.md

@@ -38,6 +38,33 @@ fn main() {
 }
 ```
 
+## Configuration
+
+You can configure test settings via a `kittest.toml` file in your workspace root.
+All possible settings and their defaults:
+```toml
+# path to the snapshot directory
+output_path = "tests/snapshots" 
+
+# default threshold for image comparison tests
+threshold = 0.6
+
+# default failed_pixel_count_threshold
+failed_pixel_count_threshold = 0
+
+[windows]
+threshold = 0.6
+failed_pixel_count_threshold = 0
+
+[macos]
+threshold = 0.6
+failed_pixel_count_threshold = 0
+
+[linux]
+threshold = 0.6
+failed_pixel_count_threshold = 0
+```
+
 ## Snapshot testing
 There is a snapshot testing feature. To create snapshot tests, enable the `snapshot` and `wgpu` features.
 Once enabled, you can call `Harness::snapshot` to render the ui and save the image to the `tests/snapshots` directory.

crates/egui_kittest/src/config.rs

@@ -0,0 +1,154 @@
+use std::io;
+use std::path::PathBuf;
+
+/// Configuration for `egui_kittest`.
+///
+/// It's loaded once (per process) by searching for a `kittest.toml` file in the project root
+/// (the directory containing `Cargo.lock`).
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct Config {
+    /// The output path for image snapshots.
+    ///
+    /// Default is "tests/snapshots" (relative to the working directory / crate root).
+    output_path: PathBuf,
+
+    /// The per-pixel threshold.
+    ///
+    /// Default is 0.6.
+    threshold: f32,
+
+    /// The number of pixels that can differ before the test is considered failed.
+    ///
+    /// Default is 0.
+    failed_pixel_count_threshold: usize,
+
+    windows: OsConfig,
+    mac: OsConfig,
+    linux: OsConfig,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            output_path: PathBuf::from("tests/snapshots"),
+            threshold: 0.6,
+            failed_pixel_count_threshold: 0,
+            windows: Default::default(),
+            mac: Default::default(),
+            linux: Default::default(),
+        }
+    }
+}
+#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct OsConfig {
+    /// Override the per-pixel threshold for this OS.
+    threshold: Option<f32>,
+
+    /// Override the failed pixel count threshold for this OS.
+    failed_pixel_count_threshold: Option<usize>,
+}
+
+fn find_kittest_toml() -> io::Result<std::path::PathBuf> {
+    let mut current_dir = std::env::current_dir()?;
+
+    loop {
+        let current_kittest = current_dir.join("kittest.toml");
+        // Check if Cargo.toml exists in this directory
+        if current_kittest.exists() {
+            return Ok(current_kittest);
+        }
+
+        // Move up one directory
+        if !current_dir.pop() {
+            return Err(io::Error::new(
+                io::ErrorKind::NotFound,
+                "kittest.toml not found",
+            ));
+        }
+    }
+}
+
+fn load_config() -> Config {
+    if let Ok(config_path) = find_kittest_toml() {
+        match std::fs::read_to_string(&config_path) {
+            Ok(config_str) => match toml::from_str(&config_str) {
+                Ok(config) => config,
+                Err(e) => panic!("Failed to parse {}: {e}", &config_path.display()),
+            },
+            Err(err) => {
+                panic!("Failed to read {}: {}", config_path.display(), err);
+            }
+        }
+    } else {
+        Config::default()
+    }
+}
+
+/// Get the global configuration.
+///
+/// See [`Config::global`] for details.
+pub fn config() -> &'static Config {
+    Config::global()
+}
+
+impl Config {
+    /// Get or load the global configuration.
+    ///
+    /// This is either
+    ///  - Based on a `kittest.toml`, found by searching from the current working directory
+    ///    (for tests that is the crate root) upwards.
+    ///  - The default [Config], if no `kittest.toml` is found.
+    pub fn global() -> &'static Self {
+        static INSTANCE: std::sync::LazyLock<Config> = std::sync::LazyLock::new(load_config);
+        &INSTANCE
+    }
+
+    /// The output path for image snapshots.
+    ///
+    /// Default is "tests/snapshots".
+    pub fn output_path(&self) -> PathBuf {
+        self.output_path.clone()
+    }
+}
+
+#[cfg(feature = "snapshot")]
+impl Config {
+    pub fn os_threshold(&self) -> crate::OsThreshold<f32> {
+        let fallback = self.threshold;
+        crate::OsThreshold {
+            windows: self.windows.threshold.unwrap_or(fallback),
+            macos: self.mac.threshold.unwrap_or(fallback),
+            linux: self.linux.threshold.unwrap_or(fallback),
+            fallback,
+        }
+    }
+
+    pub fn os_failed_pixel_count_threshold(&self) -> crate::OsThreshold<usize> {
+        let fallback = self.failed_pixel_count_threshold;
+        crate::OsThreshold {
+            windows: self
+                .windows
+                .failed_pixel_count_threshold
+                .unwrap_or(fallback),
+            macos: self.mac.failed_pixel_count_threshold.unwrap_or(fallback),
+            linux: self.linux.failed_pixel_count_threshold.unwrap_or(fallback),
+            fallback,
+        }
+    }
+
+    /// The threshold.
+    ///
+    /// Default is 1.0.
+    pub fn threshold(&self) -> f32 {
+        self.os_threshold().threshold()
+    }
+
+    /// The number of pixels that can differ before the test is considered failed.
+    ///
+    /// Default is 0.
+    pub fn failed_pixel_count_threshold(&self) -> usize {
+        self.os_failed_pixel_count_threshold().threshold()
+    }
+}

crates/egui_kittest/src/lib.rs

@@ -11,6 +11,7 @@ mod snapshot;
 pub use crate::snapshot::*;
 
 mod app_kind;
+mod config;
 mod node;
 mod renderer;
 #[cfg(feature = "wgpu")]

crates/egui_kittest/src/snapshot.rs

@@ -1,28 +1,35 @@
-use crate::Harness;
-use image::ImageError;
 use std::fmt::Display;
 use std::io::ErrorKind;
 use std::path::PathBuf;
 
+use image::ImageError;
+
+use crate::{Harness, config::config};
+
 pub type SnapshotResult = Result<(), SnapshotError>;
 
 #[non_exhaustive]
 #[derive(Clone, Debug)]
 pub struct SnapshotOptions {
     /// The threshold for the image comparison.
-    /// The default is `0.6` (which is enough for most egui tests to pass across different
+    ///
-    /// wgpu backends).
+    /// Can be configured via kittest.toml. The fallback is `0.6` (which is enough for most egui
+    /// tests to pass across different wgpu backends).
     pub threshold: f32,
 
     /// The number of pixels that can differ before the snapshot is considered a failure.
+    ///
     /// Preferably, you should use `threshold` to control the sensitivity of the image comparison.
     /// As a last resort, you can use this to allow a certain number of pixels to differ.
-    /// If `None`, the default is `0` (meaning no pixels can differ).
+    /// Can be configured via kittest.toml. The fallback is `0` (meaning no pixels can differ).
-    /// If `Some`, the value can be set per OS
     pub failed_pixel_count_threshold: usize,
 
     /// The path where the snapshots will be saved.
-    /// The default is `tests/snapshots`.
+    ///
+    /// This is relative to the current working directory (usually the crate root when
+    /// running tests).
+    ///
+    /// Can be configured via kittest.toml. The fallback is `tests/snapshots`.
     pub output_path: PathBuf,
 }
 
@@ -30,7 +37,9 @@ pub struct SnapshotOptions {
 ///
 /// This is useful if you want to set different thresholds for different operating systems.
 ///
-/// The default values are 0 / 0.0
+/// [`OsThreshold::default`] gets the default from the config file (`kittest.toml`).
+/// For `usize`, it's the `failed_pixel_count_threshold` value.
+/// For `f32`, it's the `threshold` value.
 ///
 /// Example usage:
 /// ```no_run
@@ -53,12 +62,36 @@ pub struct OsThreshold<T> {
     pub fallback: T,
 }
 
+impl Default for OsThreshold<usize> {
+    /// Returns the default `failed_pixel_count_threshold` as configured in `kittest.toml`
+    ///
+    /// The fallback is `0`.
+    fn default() -> Self {
+        config().os_failed_pixel_count_threshold()
+    }
+}
+
+impl Default for OsThreshold<f32> {
+    /// Returns the default `threshold` as configured in `kittest.toml`
+    ///
+    /// The fallback is `0.6`.
+    fn default() -> Self {
+        config().os_threshold()
+    }
+}
+
 impl From<usize> for OsThreshold<usize> {
     fn from(value: usize) -> Self {
         Self::new(value)
     }
 }
 
+impl From<f32> for OsThreshold<f32> {
+    fn from(value: f32) -> Self {
+        Self::new(value)
+    }
+}
+
 impl<T> OsThreshold<T>
 where
     T: Copy,
@@ -123,9 +156,9 @@ impl From<OsThreshold<Self>> for f32 {
 impl Default for SnapshotOptions {
     fn default() -> Self {
         Self {
-            threshold: 0.6,
+            threshold: config().threshold(),
-            output_path: PathBuf::from("tests/snapshots"),
+            output_path: config().output_path(),
-            failed_pixel_count_threshold: 0, // Default is 0, meaning no pixels can differ
+            failed_pixel_count_threshold: config().failed_pixel_count_threshold(),
         }
     }
 }

kittest.toml

@@ -0,0 +1,10 @@
+output_path = "tests/snapshots"
+
+# Other OSes get a higher threshold so they can still run tests locally without failures due to small rendering
+# differences.
+# To update snapshots, update them via ./scripts/update_snapshots_from_ci.sh or via kitdiff
+threshold = 2.0
+
+[mac]
+# Since our CI runs snapshot tests on macOS, this is our source of truth.
+threshold = 0.6