From b7da254b16ef6292ffed9c9102ff3a5a171e0303 Mon Sep 17 00:00:00 2001 From: lucasmerlin Date: Mon, 20 Apr 2026 13:10:48 +0200 Subject: [PATCH] Allow controlling the tested app --- Cargo.lock | 1 + crates/egui_demo_lib/Cargo.toml | 2 +- crates/egui_kittest/src/inspector.rs | 11 +- crates/egui_kittest/src/lib.rs | 68 +++++++--- crates/kittest_inspector/Cargo.toml | 1 + crates/kittest_inspector/src/lib.rs | 7 +- crates/kittest_inspector/src/main.rs | 194 ++++++++++++++++++++------- 7 files changed, 210 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 265628ebb..7ef47732b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2523,6 +2523,7 @@ dependencies = [ "accesskit", "bincode 2.0.1", "eframe", + "egui", "egui_extras", "serde", ] diff --git a/crates/egui_demo_lib/Cargo.toml b/crates/egui_demo_lib/Cargo.toml index d2586fa11..d19a6e433 100644 --- a/crates/egui_demo_lib/Cargo.toml +++ b/crates/egui_demo_lib/Cargo.toml @@ -55,7 +55,7 @@ serde = { workspace = true, optional = true } criterion.workspace = true egui = { workspace = true, features = ["default_fonts"] } egui_extras = { workspace = true, features = ["image", "svg"] } -egui_kittest = { workspace = true, features = ["wgpu", "snapshot"] } +egui_kittest = { workspace = true, features = ["wgpu", "snapshot", "inspector"] } image = { workspace = true, features = ["png"] } mimalloc.workspace = true # for benchmarks rand.workspace = true diff --git a/crates/egui_kittest/src/inspector.rs b/crates/egui_kittest/src/inspector.rs index 239176c0c..01a3d001b 100644 --- a/crates/egui_kittest/src/inspector.rs +++ b/crates/egui_kittest/src/inspector.rs @@ -92,15 +92,15 @@ impl Inspector { } /// Send the current frame + accesskit tree and block until the inspector replies. - /// Returns silently on send/receive failure (e.g. the inspector window was closed). + /// Returns any user events captured by the inspector (empty on failure / no control input). pub fn send_step( &mut self, image: &image::RgbaImage, pixels_per_point: f32, accesskit: Option, - ) { + ) -> Vec { if self.broken { - return; + return Vec::new(); } self.step = self.step.saturating_add(1); let frame = Frame { @@ -118,16 +118,17 @@ impl Inspector { eprintln!("egui_kittest inspector: send failed: {err}"); } self.broken = true; - return; + return Vec::new(); } match read_message::<_, InspectorReply>(&mut self.reader) { - Ok(InspectorReply::Continue) => {} + Ok(InspectorReply::Continue { events }) => events, Err(err) => { #[expect(clippy::print_stderr)] { eprintln!("egui_kittest inspector: read failed: {err}"); } self.broken = true; + Vec::new() } } } diff --git a/crates/egui_kittest/src/lib.rs b/crates/egui_kittest/src/lib.rs index cae0ef434..f6e5d1774 100644 --- a/crates/egui_kittest/src/lib.rs +++ b/crates/egui_kittest/src/lib.rs @@ -316,6 +316,22 @@ impl<'a, State> Harness<'a, State> { /// Run a single step. This will not process any events. fn _step(&mut self, sizing_pass: bool) { + self._step_inner(sizing_pass); + + #[cfg(feature = "recording")] + self.capture_frame_if_recording(false); + + // Inspector Control mode: each event the user triggers in the inspector drives one + // extra `_step_inner` so the UI re-renders, but the outer test caller stays parked + // inside this method until the inspector replies with an empty event list + // (i.e. user clicked Next/Play, or has Control off and we're just forwarding). + #[cfg(feature = "inspector")] + self.drive_inspector(); + } + + /// The core of `_step`: run egui once and update internal state. Does not touch the + /// inspector or the recording hook, so it can be called in a loop from those. + fn _step_inner(&mut self, sizing_pass: bool) { self.input.predicted_dt = self.step_dt; let mut output = self.ctx.run_ui(self.input.take(), |ui| { @@ -333,12 +349,6 @@ impl<'a, State> Harness<'a, State> { self.kittest.update(accesskit_update); self.renderer.handle_delta(&output.textures_delta); self.output = output; - - #[cfg(feature = "recording")] - self.capture_frame_if_recording(false); - - #[cfg(feature = "inspector")] - self.send_to_inspector_if_attached(); } /// Calculate the rect that includes all popups and tooltips. @@ -812,25 +822,47 @@ impl<'a, State> Harness<'a, State> { self.inspector = None; } + /// Block at the inspector until it tells us to resume, re-rendering after each batch of + /// events it sends. Events drive an internal `_step_inner` (and recording capture), but + /// do NOT return control to the outer test — the test advances only when the inspector + /// replies with no events *and* egui has no pending repaint (i.e. user hit Next/Play with + /// a settled UI, or Control mode is off). + /// + /// While paused in Control mode, any `request_repaint` from the app (animations, async + /// image loads, etc.) also drives another internal step so the user sees the UI tick. #[cfg(feature = "inspector")] - fn send_to_inspector_if_attached(&mut self) { + fn drive_inspector(&mut self) { if self.inspector.is_none() { return; } - let image = match self.render() { - Ok(img) => img, - Err(err) => { - #[expect(clippy::print_stderr)] - { - eprintln!("egui_kittest inspector: render failed, skipping frame: {err}"); + loop { + let image = match self.render() { + Ok(img) => img, + Err(err) => { + #[expect(clippy::print_stderr)] + { + eprintln!("egui_kittest inspector: render failed: {err}"); + } + return; } + }; + let tree = self.last_accesskit_update.clone(); + let ppp = self.ctx.pixels_per_point(); + let events = if let Some(inspector) = self.inspector.as_mut() { + inspector.send_step(&image, ppp, tree) + } else { + return; + }; + let wants_repaint = self.root_viewport_output().repaint_delay == Duration::ZERO; + if events.is_empty() && !wants_repaint { return; } - }; - let tree = self.last_accesskit_update.clone(); - let ppp = self.ctx.pixels_per_point(); - if let Some(inspector) = self.inspector.as_mut() { - inspector.send_step(&image, ppp, tree); + for event in events { + self.input.events.push(event); + } + self._step_inner(false); + #[cfg(feature = "recording")] + self.capture_frame_if_recording(false); } } diff --git a/crates/kittest_inspector/Cargo.toml b/crates/kittest_inspector/Cargo.toml index 804902709..6e72f3818 100644 --- a/crates/kittest_inspector/Cargo.toml +++ b/crates/kittest_inspector/Cargo.toml @@ -30,6 +30,7 @@ app = ["dep:eframe", "dep:egui_extras"] [dependencies] accesskit = { workspace = true, features = ["serde"] } bincode = { workspace = true } +egui = { workspace = true, features = ["serde"] } serde = { workspace = true } # `app` feature dependencies: diff --git a/crates/kittest_inspector/src/lib.rs b/crates/kittest_inspector/src/lib.rs index 4db1f0963..af0ba7138 100644 --- a/crates/kittest_inspector/src/lib.rs +++ b/crates/kittest_inspector/src/lib.rs @@ -41,10 +41,11 @@ pub enum HarnessMessage { } /// Sent inspector → harness in response to a [`HarnessMessage::Frame`]. -#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum InspectorReply { - /// Resume the harness (it will continue running steps and may send another frame soon). - Continue, + /// Resume the harness. `events` contains any user input captured in the inspector + /// (via Control mode) that should be queued for the next step. + Continue { events: Vec }, } const MAX_MESSAGE_BYTES: usize = 256 * 1024 * 1024; // 256 MiB sanity cap diff --git a/crates/kittest_inspector/src/main.rs b/crates/kittest_inspector/src/main.rs index 471d5220c..b035c2672 100644 --- a/crates/kittest_inspector/src/main.rs +++ b/crates/kittest_inspector/src/main.rs @@ -24,8 +24,9 @@ enum WorkerEvent { } /// UI → worker message: "you may send `Continue` to the harness now". -type ReleaseTx = mpsc::Sender<()>; -type ReleaseRx = mpsc::Receiver<()>; +/// Carries any egui events captured in Control mode that the harness should queue. +type ReleaseTx = mpsc::Sender>; +type ReleaseRx = mpsc::Receiver>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PlayState { @@ -35,7 +36,7 @@ enum PlayState { fn main() -> eframe::Result<()> { let (worker_tx, worker_rx) = mpsc::channel::(); - let (release_tx, release_rx) = mpsc::channel::<()>(); + let (release_tx, release_rx) = mpsc::channel::>(); thread::Builder::new() .name("kittest_inspector_io".into()) @@ -69,10 +70,12 @@ fn run_io(ui_tx: &mpsc::Sender, release_rx: &ReleaseRx) { if ui_tx.send(WorkerEvent::Frame(frame)).is_err() { return; } - if release_rx.recv().is_err() { + let Ok(events) = release_rx.recv() else { return; - } - if let Err(err) = write_message(&mut writer, &InspectorReply::Continue) { + }; + if let Err(err) = + write_message(&mut writer, &InspectorReply::Continue { events }) + { eprintln!("kittest_inspector: write failed: {err}"); return; } @@ -106,6 +109,10 @@ struct InspectorApp { hovered_node: Option, /// Last clicked widget (sticky). selected_node: Option, + /// When on, pointer + keyboard events are forwarded to the harness. + control_enabled: bool, + /// Events accumulated since the last release; drained when we send Continue. + queued_events: Vec, } impl InspectorApp { @@ -125,6 +132,8 @@ impl InspectorApp { connected: true, hovered_node: None, selected_node: None, + control_enabled: false, + queued_events: Vec::new(), } } @@ -137,9 +146,6 @@ impl InspectorApp { // Keep the selection sticky across frames (same NodeId may still exist). self.current_frame = Some(frame); self.worker_waiting = true; - if self.play_state == PlayState::Playing { - self.send_release(); - } } WorkerEvent::Disconnected => { self.connected = false; @@ -160,7 +166,8 @@ impl InspectorApp { if !self.worker_waiting { return; } - if self.release_tx.send(()).is_ok() { + let events = std::mem::take(&mut self.queued_events); + if self.release_tx.send(events).is_ok() { self.worker_waiting = false; } } @@ -177,6 +184,19 @@ impl eframe::App for InspectorApp { details_panel(self, ui); central_panel(self, ui); + // End-of-frame auto-release policy: + // - Control mode: stay blocked, but advance one step whenever the user generates events + // (each click / keypress = one harness step). + // - Otherwise, Playing mode runs freely; Paused mode waits for Next/Play. + let auto_release = if self.control_enabled { + !self.queued_events.is_empty() + } else { + self.play_state == PlayState::Playing + }; + if self.worker_waiting && auto_release { + self.send_release(); + } + ctx.request_repaint_after(std::time::Duration::from_millis(50)); } } @@ -185,7 +205,13 @@ fn controls_panel(app: &mut InspectorApp, ui: &mut egui::Ui) { egui::Panel::top("controls").show_inside(ui, |ui| { ui.horizontal(|ui| { let playing = app.play_state == PlayState::Playing; - if ui.selectable_label(playing, "▶ Play").clicked() { + let play_response = ui + .add_enabled_ui(!app.control_enabled, |ui| { + ui.selectable_label(playing, "▶ Play") + }) + .inner + .on_disabled_hover_text("Disabled while Control mode is on"); + if play_response.clicked() { app.play_state = PlayState::Playing; app.send_release(); } @@ -205,6 +231,17 @@ fn controls_panel(app: &mut InspectorApp, ui: &mut egui::Ui) { app.send_release(); } + ui.separator(); + + let prev_control = app.control_enabled; + ui.checkbox(&mut app.control_enabled, "🎮 Control") + .on_hover_text( + "Forward pointer and keyboard events on the rendered frame to the harness", + ); + if prev_control && !app.control_enabled { + app.queued_events.clear(); + } + ui.separator(); ui.label(format!( "frames: {} | state: {:?} | {}", @@ -371,52 +408,115 @@ fn central_panel(app: &mut InspectorApp, ui: &mut egui::Ui) { ) }; - // Hit test: smallest containing widget wins. - if let (Some(pos), Some(update)) = (response.hover_pos(), &frame.accesskit) { - let (lx, ly) = screen_to_logical(pos); - let mut best: Option<(NodeId, f64)> = None; - for (id, node) in &update.nodes { - let Some(b) = node.bounds() else { continue }; - if lx >= b.x0 && lx <= b.x1 && ly >= b.y0 && ly <= b.y1 { - let area = (b.x1 - b.x0).max(0.0) * (b.y1 - b.y0).max(0.0); - if best.is_none_or(|(_, a)| area < a) { - best = Some((*id, area)); + if app.control_enabled { + // In Control mode clicks/hovers drive the harness, not the inspector. + forward_events(app, ui, image_rect, frame.pixels_per_point, scale, &response); + } else { + // Inspection mode: hit test (smallest containing widget wins) + draw overlays. + if let (Some(pos), Some(update)) = (response.hover_pos(), &frame.accesskit) { + let (lx, ly) = screen_to_logical(pos); + let mut best: Option<(NodeId, f64)> = None; + for (id, node) in &update.nodes { + let Some(b) = node.bounds() else { continue }; + if lx >= b.x0 && lx <= b.x1 && ly >= b.y0 && ly <= b.y1 { + let area = (b.x1 - b.x0).max(0.0) * (b.y1 - b.y0).max(0.0); + if best.is_none_or(|(_, a)| area < a) { + best = Some((*id, area)); + } } } + app.hovered_node = best.map(|(id, _)| id); + } + if response.clicked() { + app.selected_node = app.hovered_node; } - app.hovered_node = best.map(|(id, _)| id); - } - if response.clicked() { - app.selected_node = app.hovered_node; - } - let painter = ui.painter_at(image_rect); - if let Some(update) = &frame.accesskit { - // Highlight selection (blue) and hover (yellow). - let draw = |id: NodeId, color: egui::Color32| { - if let Some((_, node)) = update.nodes.iter().find(|(nid, _)| *nid == id) - && let Some(b) = node.bounds() - { - painter.rect_stroke( - logical_to_screen(b), - 2.0, - egui::Stroke::new(1.5, color), - egui::StrokeKind::Outside, - ); + let painter = ui.painter_at(image_rect); + if let Some(update) = &frame.accesskit { + let draw = |id: NodeId, color: egui::Color32| { + if let Some((_, node)) = update.nodes.iter().find(|(nid, _)| *nid == id) + && let Some(b) = node.bounds() + { + painter.rect_stroke( + logical_to_screen(b), + 2.0, + egui::Stroke::new(1.5, color), + egui::StrokeKind::Outside, + ); + } + }; + if let Some(id) = app.selected_node { + draw(id, egui::Color32::from_rgb(80, 180, 255)); + } + if let Some(id) = app.hovered_node + && app.hovered_node != app.selected_node + { + draw(id, egui::Color32::from_rgb(255, 220, 90)); } - }; - if let Some(id) = app.selected_node { - draw(id, egui::Color32::from_rgb(80, 180, 255)); - } - if let Some(id) = app.hovered_node - && app.hovered_node != app.selected_node - { - draw(id, egui::Color32::from_rgb(255, 220, 90)); } } }); } +/// Inspect the inspector's own input events and forward those relevant to the harness. +/// +/// Pointer events only forward when their position is inside the rendered-image rect and their +/// coordinates are translated to harness logical space. Keyboard / text events always forward. +fn forward_events( + app: &mut InspectorApp, + ui: &egui::Ui, + image_rect: egui::Rect, + pixels_per_point: f32, + scale: f32, + image_response: &egui::Response, +) { + let to_logical = |pos: egui::Pos2| -> egui::Pos2 { + let f = pixels_per_point * scale; + egui::pos2( + (pos.x - image_rect.min.x) / f, + (pos.y - image_rect.min.y) / f, + ) + }; + + let input_events = ui.ctx().input(|i| i.events.clone()); + for ev in input_events { + match ev { + egui::Event::PointerMoved(pos) if image_rect.contains(pos) => { + app.queued_events + .push(egui::Event::PointerMoved(to_logical(pos))); + } + egui::Event::PointerButton { + pos, + button, + pressed, + modifiers, + } if image_rect.contains(pos) => { + app.queued_events.push(egui::Event::PointerButton { + pos: to_logical(pos), + button, + pressed, + modifiers, + }); + } + egui::Event::PointerGone => { + app.queued_events.push(egui::Event::PointerGone); + } + mw @ egui::Event::MouseWheel { .. } if image_response.hovered() => { + app.queued_events.push(mw); + } + ev @ (egui::Event::Text(_) + | egui::Event::Key { .. } + | egui::Event::Copy + | egui::Event::Cut + | egui::Event::Paste(_) + | egui::Event::Ime(_)) => { + app.queued_events.push(ev); + } + _ => {} + } + } +} + fn kv_grid(ui: &mut egui::Ui, id: &str, body: impl FnOnce(&mut egui::Ui)) { egui::Grid::new(id) .num_columns(2)