From 622218e94f6dc43f66eae27e8e0dc5577486c621 Mon Sep 17 00:00:00 2001 From: lucasmerlin Date: Thu, 21 May 2026 10:57:42 +0200 Subject: [PATCH] Add egui_inspection crate and eframe inspection hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New `egui_inspection` crate ships: - `protocol` (default): wire types + length-prefixed msgpack framing for the inspector ↔ egui-peer connection. Transport-neutral (stdio / unix socket / TCP). - `plugin`: `InspectionPlugin`, an `egui::Plugin` that dials a unix socket from `EGUI_INSPECTION_SOCKET`, streams frames + accesskit tree updates, and applies inbound `InspectorCommand`s back into the running `egui::Context`. eframe gains an `inspection` feature that auto-attaches the plugin during native startup (glow + wgpu integrations) when the env var is set. Connection failures log via `log::warn!` and do not abort startup. Lives in its own crate (rather than `egui_kittest`) so eframe can pull the protocol in without picking up the test harness, and so external tools can depend on it directly. --- Cargo.lock | 42 +++ Cargo.toml | 3 + crates/eframe/Cargo.toml | 7 + crates/eframe/src/lib.rs | 27 ++ crates/eframe/src/native/glow_integration.rs | 2 + crates/eframe/src/native/wgpu_integration.rs | 3 + crates/egui_inspection/Cargo.toml | 45 +++ crates/egui_inspection/README.md | 15 + crates/egui_inspection/src/lib.rs | 29 ++ crates/egui_inspection/src/plugin.rs | 356 +++++++++++++++++++ crates/egui_inspection/src/protocol.rs | 280 +++++++++++++++ 11 files changed, 809 insertions(+) create mode 100644 crates/egui_inspection/Cargo.toml create mode 100644 crates/egui_inspection/README.md create mode 100644 crates/egui_inspection/src/lib.rs create mode 100644 crates/egui_inspection/src/plugin.rs create mode 100644 crates/egui_inspection/src/protocol.rs diff --git a/Cargo.lock b/Cargo.lock index 94bfe892a..71f844172 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1219,6 +1219,7 @@ dependencies = [ "egui-wgpu", "egui-winit", "egui_glow", + "egui_inspection", "glow", "glutin", "glutin-winit", @@ -1391,6 +1392,18 @@ dependencies = [ "winit", ] +[[package]] +name = "egui_inspection" +version = "0.34.2" +dependencies = [ + "document-features", + "egui", + "image", + "rmp-serde", + "serde", + "serde_bytes", +] + [[package]] name = "egui_kittest" version = "0.34.2" @@ -3947,6 +3960,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "ron" version = "0.12.0" @@ -4144,6 +4176,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index 3710f8cb3..e46fbb611 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/egui_demo_lib", "crates/egui_extras", "crates/egui_glow", + "crates/egui_inspection", "crates/egui_kittest", "crates/egui-wgpu", "crates/egui-winit", @@ -65,6 +66,7 @@ egui_extras = { version = "0.34.2", path = "crates/egui_extras", default-feature egui-wgpu = { version = "0.34.2", path = "crates/egui-wgpu", default-features = false } egui_demo_lib = { version = "0.34.2", path = "crates/egui_demo_lib", default-features = false } egui_glow = { version = "0.34.2", path = "crates/egui_glow", default-features = false } +egui_inspection = { version = "0.34.2", path = "crates/egui_inspection", default-features = false } egui_kittest = { version = "0.34.2", path = "crates/egui_kittest", default-features = false } eframe = { version = "0.34.2", path = "crates/eframe", default-features = false } @@ -123,6 +125,7 @@ raw-window-handle = "0.6.2" rayon = "1.11.0" resvg = { version = "0.45.1", default-features = false } rfd = "0.17.2" +rmp-serde = "1.3.1" ron = "0.12.0" self_cell = "1.2.1" serde = { version = "1.0.228", features = ["derive"] } diff --git a/crates/eframe/Cargo.toml b/crates/eframe/Cargo.toml index 0530b64a8..a20dc1512 100644 --- a/crates/eframe/Cargo.toml +++ b/crates/eframe/Cargo.toml @@ -116,6 +116,12 @@ x11 = [ ## This is used to generate images for examples. __screenshot = [] +## Enable the [`egui_inspection`] plugin. When the `EGUI_INSPECTION_SOCKET` env var points +## at a unix socket, eframe attaches an `InspectionPlugin` to the egui context on startup +## that streams the AccessKit tree and applies received commands. Unix-only; no-op on +## non-unix targets. +inspection = ["dep:egui_inspection", "accesskit"] + [dependencies] egui = { workspace = true, default-features = false, features = ["bytemuck"] } @@ -131,6 +137,7 @@ web-time.workspace = true # Optional dependencies egui_glow = { workspace = true, optional = true, default-features = false } +egui_inspection = { workspace = true, optional = true, features = ["plugin"] } glow = { workspace = true, optional = true } ron = { workspace = true, optional = true, features = ["integer128"] } serde = { workspace = true, optional = true } diff --git a/crates/eframe/src/lib.rs b/crates/eframe/src/lib.rs index e5247a037..35d68e231 100644 --- a/crates/eframe/src/lib.rs +++ b/crates/eframe/src/lib.rs @@ -209,6 +209,33 @@ pub use native::file_storage::storage_dir; #[cfg(not(target_arch = "wasm32"))] pub mod icon_data; +// ---------------------------------------------------------------------------- + +/// Attach an [`egui_inspection::InspectionPlugin`] to `ctx` if the +/// `EGUI_INSPECTION_SOCKET` env var points at a reachable unix socket. +/// +/// No-op when: +/// - the `inspection` feature isn't enabled, +/// - the target isn't unix, or +/// - the env var is unset. +/// +/// Connection failures are logged via `log::warn!` but do not abort startup — running +/// without an inspector is always valid. +#[cfg(all(feature = "inspection", unix, not(target_arch = "wasm32")))] +pub(crate) fn maybe_attach_inspection_plugin(ctx: &egui::Context, label: Option) { + match egui_inspection::InspectionPlugin::from_env(label) { + Ok(Some(plugin)) => { + log::info!("eframe: attaching egui_inspection plugin"); + ctx.add_plugin(plugin); + } + Ok(None) => {} + Err(err) => log::warn!("eframe: egui_inspection attach failed: {err}"), + } +} + +#[cfg(not(all(feature = "inspection", unix, not(target_arch = "wasm32"))))] +pub(crate) fn maybe_attach_inspection_plugin(_ctx: &egui::Context, _label: Option) {} + /// This is how you start a native (desktop) app. /// /// The first argument is name of your app, which is an identifier diff --git a/crates/eframe/src/native/glow_integration.rs b/crates/eframe/src/native/glow_integration.rs index 9074507a5..1a2849a84 100644 --- a/crates/eframe/src/native/glow_integration.rs +++ b/crates/eframe/src/native/glow_integration.rs @@ -299,6 +299,8 @@ impl<'app> GlowWinitApp<'app> { let app_creator = std::mem::take(&mut self.app_creator) .expect("Single-use AppCreator has unexpectedly already been taken"); + crate::maybe_attach_inspection_plugin(&integration.egui_ctx, Some(self.app_name.clone())); + let app: Box = { // Use latest raw_window_handle for eframe compatibility use raw_window_handle::{HasDisplayHandle as _, HasWindowHandle as _}; diff --git a/crates/eframe/src/native/wgpu_integration.rs b/crates/eframe/src/native/wgpu_integration.rs index 59490bdb2..e68cef737 100644 --- a/crates/eframe/src/native/wgpu_integration.rs +++ b/crates/eframe/src/native/wgpu_integration.rs @@ -291,6 +291,9 @@ impl<'app> WgpuWinitApp<'app> { let app_creator = std::mem::take(&mut self.app_creator) .expect("Single-use AppCreator has unexpectedly already been taken"); + + crate::maybe_attach_inspection_plugin(&egui_ctx, Some(self.app_name.clone())); + let cc = CreationContext { egui_ctx: egui_ctx.clone(), integration_info: integration.frame.info().clone(), diff --git a/crates/egui_inspection/Cargo.toml b/crates/egui_inspection/Cargo.toml new file mode 100644 index 000000000..b619bc464 --- /dev/null +++ b/crates/egui_inspection/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "egui_inspection" +version.workspace = true +authors = [ + "Lucas Meurer ", + "Emil Ernerfeldt ", +] +description = "Wire protocol and egui::Plugin for live inspection of running egui apps and kittest harnesses" +edition.workspace = true +rust-version.workspace = true +homepage = "https://github.com/emilk/egui" +license.workspace = true +readme = "./README.md" +repository = "https://github.com/emilk/egui" +categories = ["gui", "development-tools::testing", "accessibility"] +keywords = ["gui", "egui", "inspector", "accesskit", "testing"] +include = ["../../LICENSE-APACHE", "../../LICENSE-MIT", "**/*.rs", "Cargo.toml"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--generate-link-to-definition"] + +[features] +default = ["protocol"] + +## Wire-protocol types (`HarnessMessage`, `InspectorCommand`, …) plus length-prefixed +## MessagePack framing helpers. No `egui` dependency beyond `egui::accesskit`. +protocol = ["dep:rmp-serde", "dep:serde", "dep:serde_bytes", "egui/serde"] + +## `InspectionPlugin` — an `egui::Plugin` impl that streams frames + accesskit tree to +## an inspector over a unix socket and applies received commands. Auto-attaches when +## the [`INSPECTION_SOCKET_ENV_VAR`] env var is set. +plugin = ["protocol", "dep:image"] + +[dependencies] +egui.workspace = true +serde = { workspace = true, optional = true } +serde_bytes = { version = "0.11.17", optional = true } +rmp-serde = { workspace = true, optional = true } +image = { workspace = true, optional = true } + +document-features = { workspace = true, optional = true } + +[lints] +workspace = true diff --git a/crates/egui_inspection/README.md b/crates/egui_inspection/README.md new file mode 100644 index 000000000..c5ce75b84 --- /dev/null +++ b/crates/egui_inspection/README.md @@ -0,0 +1,15 @@ +# egui_inspection + +Wire protocol and `egui::Plugin` for live inspection of running egui apps and +kittest harnesses. + +Two layers: + +- **`protocol`** (default feature): length-prefixed MessagePack messages used by + `egui_kittest`'s inspector, the external `kittest_inspector` UI, and the + `egui_kittest_mcp` server. + +- **`plugin`** (opt-in): an `egui::Plugin` implementation that streams frames + + AccessKit tree updates to an inspector over a unix domain socket and applies + received `InspectorCommand`s back into the running app. Auto-attaches when + `EGUI_INSPECTION_SOCKET` is set. diff --git a/crates/egui_inspection/src/lib.rs b/crates/egui_inspection/src/lib.rs new file mode 100644 index 000000000..2ef64e2f0 --- /dev/null +++ b/crates/egui_inspection/src/lib.rs @@ -0,0 +1,29 @@ +#![cfg_attr(doc, doc = include_str!("../README.md"))] +//! +//! ## Feature flags +#![cfg_attr(feature = "document-features", doc = document_features::document_features!())] + +#[cfg(feature = "protocol")] +pub mod protocol; + +#[cfg(feature = "protocol")] +pub use protocol::{ + Capabilities, Frame, FrameScreenshot, HarnessMessage, InspectorCommand, MAX_MESSAGE_BYTES, + PROTOCOL_VERSION, PeerHello, PeerKind, SourceView, read_message, write_message, +}; + +/// Environment variable: when set to a unix socket path, [`InspectionPlugin::from_env`] +/// (and similar inspector-side code) connects to it. +/// +/// Exposed unconditionally so both ends of the connection — the plugin (on `plugin`, +/// unix) and the inspector / MCP server — can reference the same name without pulling in +/// the full plugin impl. +pub const INSPECTION_SOCKET_ENV_VAR: &str = "EGUI_INSPECTION_SOCKET"; + +// The plugin uses `std::os::unix::net::UnixStream` for transport, so the impl is +// unix-only. Non-unix builds with `plugin` enabled still get the protocol types. +#[cfg(all(feature = "plugin", unix))] +mod plugin; + +#[cfg(all(feature = "plugin", unix))] +pub use plugin::{InspectionError, InspectionPlugin}; diff --git a/crates/egui_inspection/src/plugin.rs b/crates/egui_inspection/src/plugin.rs new file mode 100644 index 000000000..d8403be4d --- /dev/null +++ b/crates/egui_inspection/src/plugin.rs @@ -0,0 +1,356 @@ +//! [`InspectionPlugin`] — an [`egui::Plugin`] that streams frames + AccessKit tree updates +//! to an inspector over a unix domain socket and applies received commands back into the +//! running app. +//! +//! Connection model: +//! - The inspector binds a unix socket. The egui peer dials it. +//! - The plugin spawns one reader thread and one writer thread, each owning one half of the +//! stream. UI-thread hooks (`input_hook` / `output_hook`) only touch in-process channels +//! and the reader-side command queue. +//! - If the writer channel is saturated, the plugin drops the oldest frame in favor of the +//! newest so the UI thread never blocks on a slow inspector. +//! +//! Live apps don't own a deterministic run loop, so `Step` / `Run` / `Play` / `Pause` +//! commands are no-ops. `Handle { events }` is honored by appending the events to the next +//! `RawInput`. After every received command the reader thread calls +//! `Context::request_repaint` so the integration wakes up even when the UI is otherwise +//! idle — without this, queued events would sit in the channel until the next mouse move. +//! +//! # Reference cycle +//! +//! The plugin holds a clone of `egui::Context` so the reader thread can wake the UI loop. +//! `egui::Context` is `Arc>` and the context owns its plugins, so this creates an +//! intentional cycle: the context will not drop until the process exits. Acceptable for a +//! live-debugging inspector — the typical workflow is "attach for the lifetime of the +//! process, then exit." For deterministic shutdown, kill the process. + +use std::io::{BufReader, BufWriter}; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::mpsc; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread; + +use egui::{Context, FullOutput, RawInput}; + +use crate::INSPECTION_SOCKET_ENV_VAR; +use crate::protocol::{ + Capabilities, Frame, FrameScreenshot, HarnessMessage, InspectorCommand, PROTOCOL_VERSION, + PeerHello, PeerKind, read_message, write_message, +}; + +/// Errors that can occur attaching to an inspector. +#[derive(Debug)] +pub enum InspectionError { + /// Failed to dial the inspector socket. + Connect(std::io::Error), + /// Failed to set up reader / writer threads. + Pipe(String), +} + +impl std::fmt::Display for InspectionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Connect(err) => write!( + f, + "failed to connect to egui_inspection socket (set {INSPECTION_SOCKET_ENV_VAR}): {err}" + ), + Self::Pipe(msg) => write!(f, "egui_inspection pipe setup failed: {msg}"), + } + } +} + +impl std::error::Error for InspectionError {} + +/// Bounded outbound queue depth. If the inspector falls behind we drop oldest frames +/// rather than block the UI thread. +const OUTBOUND_QUEUE_DEPTH: usize = 8; + +/// Shared between [`InspectionPlugin::setup`] and the reader thread so the reader can wake +/// the UI loop after each received command. Written exactly once in `setup`. +type SharedCtx = Arc>; + +/// `egui::Plugin` that streams the running app's state to an inspector. +pub struct InspectionPlugin { + /// Incoming commands from the inspector. + command_rx: Arc>>, + /// Outbound messages → writer thread → socket. Bounded; oldest is dropped on overflow. + outbound_tx: mpsc::SyncSender, + /// Filled in `Plugin::setup`; read by the reader thread to call `request_repaint` after + /// every received command. + shared_ctx: SharedCtx, + /// Monotonic frame counter. + step: u64, + /// Frame data (accesskit + meta) captured in `output_hook`, held until the matching + /// `Event::Screenshot` arrives in the next `input_hook`. Emitting only on pair-up keeps + /// the inspector's screenshot and accesskit tree in lockstep — the alternative (emit + /// accesskit now, screenshot later) shows widget boxes that don't match the rendered + /// frame they overlay. + pending_frame: Option, + /// `true` between dispatching `ViewportCommand::Screenshot` and observing the reply + /// `Event::Screenshot`. While set, the plugin keeps requesting repaints so the + /// integration eventually paints a visible frame and the screenshot fulfills (the eframe + /// wgpu path skips capture when the viewport reports `visible=false`). + awaiting_screenshot: bool, + /// Set by [`InspectorCommand::Screenshot`]; consumed by the next `output_hook` which + /// dispatches a `ViewportCommand::Screenshot` and stashes the frame. + one_shot_screenshot: bool, + /// When `true`, every `output_hook` requests a `ViewportCommand::Screenshot` and holds + /// the frame until the screenshot returns. Toggled by + /// [`InspectorCommand::SetContinuousScreenshots`]. + continuous_screenshots: bool, + /// Background threads — held so they live as long as the plugin. + _reader_thread: thread::JoinHandle<()>, + _writer_thread: thread::JoinHandle<()>, +} + +impl InspectionPlugin { + /// If [`INSPECTION_SOCKET_ENV_VAR`] is set, return a plugin connected to it. + /// Returns `Ok(None)` when the env var is unset. + /// + /// # Errors + /// When the env var is set but the socket can't be dialed. + pub fn from_env(label: Option) -> Result, InspectionError> { + let Ok(path) = std::env::var(INSPECTION_SOCKET_ENV_VAR) else { + return Ok(None); + }; + Self::attach(PathBuf::from(path), label).map(Some) + } + + /// Dial the given unix socket and attach. + /// + /// # Errors + /// When the socket can't be dialed or a thread can't be spawned. + pub fn attach(socket_path: PathBuf, label: Option) -> Result { + let stream = UnixStream::connect(&socket_path).map_err(InspectionError::Connect)?; + let reader_stream = stream + .try_clone() + .map_err(InspectionError::Connect)?; + let writer_stream = stream; + + let shared_ctx: SharedCtx = Arc::new(OnceLock::new()); + + let (command_tx, command_rx) = mpsc::channel::(); + let reader_ctx = shared_ctx.clone(); + let reader_thread = thread::Builder::new() + .name("egui_inspection_reader".into()) + .spawn(move || run_reader(BufReader::new(reader_stream), &command_tx, &reader_ctx)) + .map_err(|err| InspectionError::Pipe(format!("spawn reader thread: {err}")))?; + + let (outbound_tx, outbound_rx) = mpsc::sync_channel::(OUTBOUND_QUEUE_DEPTH); + let writer_thread = thread::Builder::new() + .name("egui_inspection_writer".into()) + .spawn(move || run_writer(BufWriter::new(writer_stream), outbound_rx)) + .map_err(|err| InspectionError::Pipe(format!("spawn writer thread: {err}")))?; + + // Hello must be the first message on the wire. Send via the writer-thread queue + // (rather than directly on the stream) so ordering against later frames is + // preserved even under contention. + let hello = HarnessMessage::Hello(PeerHello { + protocol_version: PROTOCOL_VERSION, + peer_kind: PeerKind::Live, + capabilities: Capabilities::LIVE, + // Live apps start accesskit-only; inspector flips on via + // `SetContinuousScreenshots(true)` when it wants images. + continuous_screenshots: false, + label, + }); + outbound_tx + .send(hello) + .map_err(|err| InspectionError::Pipe(format!("send Hello: {err}")))?; + + Ok(Self { + command_rx: Arc::new(Mutex::new(command_rx)), + outbound_tx, + shared_ctx, + step: 0, + pending_frame: None, + awaiting_screenshot: false, + one_shot_screenshot: false, + continuous_screenshots: false, + _reader_thread: reader_thread, + _writer_thread: writer_thread, + }) + } + + /// Best-effort send. Drops oldest frame on overflow so the UI thread never blocks. + fn send(&self, msg: HarnessMessage) { + match self.outbound_tx.try_send(msg) { + Ok(()) => {} + Err(mpsc::TrySendError::Full(msg)) => { + // Queue saturated — try once more in case the writer just drained a slot. + // If still full we drop the message. UI thread never blocks. + let _ = self.outbound_tx.try_send(msg); + } + Err(mpsc::TrySendError::Disconnected(_)) => { /* writer is gone */ } + } + } +} + +impl egui::Plugin for InspectionPlugin { + fn debug_name(&self) -> &'static str { + "egui_inspection" + } + + fn setup(&mut self, ctx: &Context) { + // We rely on the AccessKit tree to describe the UI structure to the inspector. + ctx.enable_accesskit(); + // Hand the context to the reader thread so it can wake the UI loop when commands + // arrive on an otherwise-idle app. `set` only succeeds the first time, which is + // what we want — `setup` is documented to run once per plugin registration. + let _ = self.shared_ctx.set(ctx.clone()); + } + + fn input_hook(&mut self, input: &mut RawInput) { + // Capture any screenshot reply the integration produced in response to our previous + // `ViewportCommand::Screenshot`. If we're holding a frame waiting for this + // screenshot, attach the pixels and emit the pair now. Without a pending frame the + // screenshot is stray (we never dispatched) and we drop it. We observe (don't + // consume) — apps using the same event keep getting it. + for ev in &input.events { + if let egui::Event::Screenshot { image, .. } = ev { + self.awaiting_screenshot = false; + if let Some(mut frame) = self.pending_frame.take() { + let [w, h] = [image.size[0] as u32, image.size[1] as u32]; + let rgba: Vec = image.pixels.iter().flat_map(|c| c.to_array()).collect(); + frame.screenshot = Some(FrameScreenshot { + width: w, + height: h, + rgba, + }); + self.send(HarnessMessage::Frame(Box::new(frame))); + } + break; + } + } + + // Drain any commands the inspector sent since the previous frame. + let mut got_command = false; + let rx = self.command_rx.lock().expect("poisoned"); + while let Ok(cmd) = rx.try_recv() { + got_command = true; + match cmd { + InspectorCommand::Handle { events } => { + input.events.extend(events); + } + InspectorCommand::Screenshot => { + self.one_shot_screenshot = true; + } + InspectorCommand::SetContinuousScreenshots(on) => { + self.continuous_screenshots = on; + } + InspectorCommand::Resize { width, height } => { + if let Some(ctx) = self.shared_ctx.get() { + ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(egui::vec2( + width as f32, + height as f32, + ))); + } + } + // The live-app path doesn't own a deterministic run loop, so the + // step/run/play/pause commands are no-ops here. The deterministic side + // lives in `egui_kittest::InspectorPlugin`. + InspectorCommand::Step + | InspectorCommand::Run + | InspectorCommand::Play + | InspectorCommand::Pause => {} + } + } + + // Reactive-mode apps only paint on input. The reader thread's `request_repaint` + // woke us for the current frame, but viewport-command replies (`Event::Screenshot`) + // and synthetic `Handle` events both need at least one *more* frame to be observed + // by the host app and round-trip back into a `Frame` we can emit. Without an extra + // repaint scheduled now, the app goes idle until an unrelated wake-up (mouse move, + // timer) and the inspector sees a multi-second stall. + // + // While a screenshot is outstanding (or continuous mode is on), keep requesting + // repaints every frame — eframe's wgpu path skips screenshot capture when the + // viewport reports `visible=false`, so a backgrounded window won't fulfill the + // request until it next becomes visible. We can't force visibility from here without + // disturbing focus, but pumping repaints keeps the app alive so the moment the OS + // reports visibility (cursor enters, app brought forward, system unhide) the queued + // action fires. + if got_command || self.awaiting_screenshot || self.continuous_screenshots { + if let Some(ctx) = self.shared_ctx.get() { + ctx.request_repaint(); + } + } + } + + fn output_hook(&mut self, output: &mut FullOutput) { + self.step = self.step.saturating_add(1); + let want_screenshot = self.continuous_screenshots || self.one_shot_screenshot; + self.one_shot_screenshot = false; + + // Pull the AccessKit tree update out of the PlatformOutput. We *clone* rather than + // take so the host integration still receives it for the real accessibility stack. + let tree = output.platform_output.accesskit_update.clone(); + + let frame = Frame { + step: self.step, + pixels_per_point: output.pixels_per_point, + screenshot: None, + accesskit: tree, + source: None, + }; + + if !want_screenshot { + // No screenshot needed — emit immediately. + self.send(HarnessMessage::Frame(Box::new(frame))); + return; + } + + // Want a screenshot. If the previous frame's request is still outstanding, drop + // this output entirely (the screenshot reply would otherwise pair with a stale + // accesskit tree). Slow inspector → matched-pair frames > throughput; the user + // explicitly opted into this delay by enabling continuous screenshots. + if self.awaiting_screenshot { + return; + } + + // Hold the frame; dispatch a screenshot request for what was just rendered. The + // matching `Event::Screenshot` arrives in the next `input_hook`, where we attach + // pixels and emit. + self.pending_frame = Some(frame); + if let Some(ctx) = self.shared_ctx.get() { + ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::default())); + self.awaiting_screenshot = true; + } + } +} + +/// Reader-thread entry point: forward every decoded [`InspectorCommand`] into the channel +/// until EOF or the receiver is dropped. After each enqueue, wake the UI thread so an +/// otherwise-idle app actually processes the command on its next frame. +fn run_reader( + mut reader: BufReader, + tx: &mpsc::Sender, + ctx: &SharedCtx, +) { + loop { + match read_message::<_, InspectorCommand>(&mut reader) { + Ok(cmd) => { + if tx.send(cmd).is_err() { + return; + } + if let Some(ctx) = ctx.get() { + ctx.request_repaint(); + } + } + Err(_) => return, + } + } +} + +/// Writer-thread entry point: drain the outbound queue, framing each message to the socket. +fn run_writer( + mut writer: BufWriter, + rx: mpsc::Receiver, +) { + while let Ok(msg) = rx.recv() { + if write_message(&mut writer, &msg).is_err() { + return; + } + } +} diff --git a/crates/egui_inspection/src/protocol.rs b/crates/egui_inspection/src/protocol.rs new file mode 100644 index 000000000..c5468371c --- /dev/null +++ b/crates/egui_inspection/src/protocol.rs @@ -0,0 +1,280 @@ +//! Wire protocol shared between an egui peer (an `egui_kittest::Harness` or a live +//! `eframe` app running [`crate::InspectionPlugin`]) and an external inspector +//! (the standalone `kittest_inspector` UI binary, or the `egui_kittest_mcp` server). +//! +//! The egui peer writes [`HarnessMessage`]s (frames plus blocking-state updates) into the +//! transport. The inspector writes [`InspectorCommand`]s back to drive the peer. Shutdown +//! is detected on either side via EOF — no explicit goodbye message. +//! +//! Messages are framed as a 4-byte big-endian length followed by a MessagePack-encoded body +//! (`rmp-serde`). Transport-neutral: the same framing works on stdio, unix sockets, and TCP. +//! +//! Living in its own crate (rather than `egui_kittest`) lets eframe pull the protocol in +//! without picking up the test harness, and lets external tools depend on it directly. + +use std::io::{self, Read, Write}; + +use egui::accesskit; + +/// Wire-protocol version sent in [`PeerHello::protocol_version`]. Bump whenever a +/// non-additive change is made to [`HarnessMessage`] / [`InspectorCommand`] / their +/// payload structs. The inspector should refuse peers with a higher major version than +/// it understands. +pub const PROTOCOL_VERSION: u32 = 1; + +/// What kind of egui peer the inspector is talking to. Determines which controls the +/// inspector UI should render (Step / Pause buttons make no sense against a live app). +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum PeerKind { + /// A deterministic `egui_kittest::Harness` — supports stepping, pause/play, panic + /// capture, and source highlighting. + Kittest, + /// A live `eframe` app running [`crate::InspectionPlugin`] — no deterministic run + /// loop, no panic capture, no source view. + Live, +} + +/// Which optional [`InspectorCommand`] variants the peer honors. The inspector should +/// hide / disable UI for commands whose capability is `false`. +/// +/// `Handle` is always supported (no flag) — every peer accepts event injection. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Capabilities { + /// Peer honors [`InspectorCommand::Step`]. + pub step: bool, + /// Peer honors [`InspectorCommand::Run`]. + pub run: bool, + /// Peer honors [`InspectorCommand::Play`] / [`InspectorCommand::Pause`]. + pub play_pause: bool, + /// Peer honors [`InspectorCommand::Screenshot`]. + pub screenshot: bool, + /// Peer honors [`InspectorCommand::SetContinuousScreenshots`] — i.e. it can be asked to + /// attach a fresh [`FrameScreenshot`] to every outgoing [`Frame`] until told to stop. + pub continuous_screenshots: bool, + /// Peer honors [`InspectorCommand::Resize`]. + pub resize: bool, +} + +impl Capabilities { + /// Capabilities of a deterministic kittest harness: all execution-control commands plus + /// both one-shot and continuous screenshot modes. The harness ships with continuous on + /// by default (matching the pre-flag behavior of always-fresh frames); the inspector + /// can flip it off via [`InspectorCommand::SetContinuousScreenshots`]`(false)` to skip + /// the per-step render cost when it only needs the accesskit tree. + pub const KITTEST: Self = Self { + step: true, + run: true, + play_pause: true, + screenshot: true, + continuous_screenshots: true, + resize: true, + }; + + /// Capabilities of a live `eframe` app: no execution-control (no own run loop), but + /// the integration honors viewport-level screenshot and resize requests, and the + /// plugin can be flipped into per-frame screenshot mode. + pub const LIVE: Self = Self { + step: false, + run: false, + play_pause: false, + screenshot: true, + continuous_screenshots: true, + resize: true, + }; +} + +/// First [`HarnessMessage`] sent on every connection. Identifies the peer and declares +/// which optional commands it will honor. The inspector should treat the absence of a +/// `Hello` (i.e. a `Frame` arriving first) as a protocol error. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct PeerHello { + /// [`PROTOCOL_VERSION`] of the peer. + pub protocol_version: u32, + pub peer_kind: PeerKind, + pub capabilities: Capabilities, + /// Whether the peer starts in continuous-screenshot mode (i.e. attaches a + /// [`FrameScreenshot`] to every `Frame` until told otherwise). Inspectors should treat + /// this as the authoritative initial state rather than relying on per-peer defaults. + /// Only meaningful when [`Capabilities::continuous_screenshots`] is `true`. + pub continuous_screenshots: bool, + /// Human-readable identifier (test name, app name). Replaces the per-`Frame` label. + pub label: Option, +} + +/// One source file plus the test-source lines the inspector should highlight inside it. +/// +/// The harness captures `#[track_caller]` locations for the `.run()`/`.step()` call that +/// produced the frame and for each event consumed by it. The inspector highlights +/// [`Self::call_site_line`] for the runner call and [`Self::event_lines`] for each event. +/// +/// Only populated by `egui_kittest`. Live apps (via [`crate::InspectionPlugin`]) leave this +/// `None` on every [`Frame`] — they have no test source to point at. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SourceView { + /// Absolute or crate-relative path as reported by `std::panic::Location::file`. + pub path: String, + /// Entire file contents, lines separated by `\n`. `None` if the file couldn't be read. + pub contents: Option, + /// Line number of the `.run()` / `.step()` call that produced this frame. + pub call_site_line: Option, + /// Line numbers of events consumed by this frame's step, in queue order. + pub event_lines: Vec, + /// Line number of a panic captured in this file. The inspector highlights this line in + /// red. Set on the [`HarnessMessage::Finished`] source view when a panic was captured. + pub panic_line: Option, +} + +/// Rendered framebuffer attached to a [`Frame`]. Absent on accesskit-only frames (live +/// apps default to "tree-only" until the inspector asks for screenshots). +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct FrameScreenshot { + /// Image width in physical pixels. + pub width: u32, + /// Image height in physical pixels. + pub height: u32, + /// Tightly packed RGBA8 pixels (length = `width * height * 4`). `serde_bytes` encodes + /// this as a msgpack `bin` blob (one type tag + raw bytes) instead of the default + /// `Vec` path of one type tag *per byte*, which would roughly double on-wire size. + #[serde(with = "serde_bytes")] + pub rgba: Vec, +} + +/// A single update from the egui peer: accesskit tree + optional screenshot. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Frame { + /// Monotonically increasing step counter. + pub step: u64, + /// `physical_pixel = logical_point * pixels_per_point`. AccessKit bounds are in logical + /// coords, the screenshot is in physical pixels — multiply by this to align them. + pub pixels_per_point: f32, + /// Rendered framebuffer for this step, when available. `None` for live-app frames + /// outside continuous-screenshot mode that didn't receive an `Event::Screenshot` reply + /// (i.e. accesskit-only updates). Kittest harnesses populate this on every frame. + pub screenshot: Option, + /// Latest accesskit tree update, if any. + pub accesskit: Option, + /// The test source file associated with this frame + the lines to highlight inside it. + /// `None` for live apps. + pub source: Option, +} + +/// Sent egui-peer → inspector. Always begins with a single [`Self::Hello`]. After that, +/// frames carry rendered images; `Blocked` signals when the harness's blocking state +/// changes without a visual update (e.g. at `after_run`, where nothing has re-rendered +/// since the last `after_step`). Live apps never send `Blocked`. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum HarnessMessage { + /// Identifies the peer and declares its capabilities. Sent exactly once, as the very + /// first message on the connection, before any [`Self::Frame`]. + Hello(PeerHello), + /// A new frame (image + tree + source) is available. + Frame(Box), + /// The peer is now either blocked (`true`) waiting for an [`InspectorCommand`], or + /// running freely (`false`). + Blocked(bool), + /// The test has ended. Implies [`Self::Blocked`]`(true)`: the harness blocks after + /// sending this, and any subsequent `Step` / `Run` / `Play` command dismisses the result + /// and lets the harness drop. + /// + /// Live apps never send this. + Finished { + /// `true` on pass; `false` if a panic was in progress when the harness dropped. + ok: bool, + /// Panic message, if captured (requires `egui_kittest::install_panic_hook()`). + message: Option, + /// Final-frame source context: the test entry point's file, with the panic line (if + /// any and if it matches that file) recorded in [`SourceView::panic_line`]. + source: Option, + }, +} + +/// Sent inspector → egui peer at any time to drive execution. +/// +/// `egui_kittest` blocks at `after_step` / `after_run` hooks (and at those hooks only). +/// Which command it waits for, and whether it returns to blocking after executing one, +/// depends on the command that last arrived — see each variant's docs. +/// +/// Live apps (via [`crate::InspectionPlugin`]) treat `Step` / `Run` / `Play` / `Pause` as +/// no-ops — they don't own a deterministic run loop. `Handle` is honored on the next frame. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum InspectorCommand { + /// Advance one frame, then block at the next `after_step`. + Step, + /// Run until the next `after_run` hook fires, then block. + Run, + /// Run freely until a [`Self::Pause`], [`Self::Step`], or [`Self::Run`] command arrives. + /// Frames keep streaming while playing — the inspector may send [`Self::Handle`] at any + /// point without interrupting play. + Play, + /// Cancel [`Self::Play`] (no-op when already blocked). + Pause, + /// Queue these events on the peer and run a single step. Does not change the peer's + /// Pause / Play / Run state. + Handle { events: Vec }, + /// Request a full-framebuffer screenshot for the next frame. + /// + /// Live apps (via [`crate::InspectionPlugin`]) issue a + /// [`egui::ViewportCommand::Screenshot`], intercept the resulting + /// [`egui::Event::Screenshot`], and emit a [`HarnessMessage::Frame`] with + /// [`Frame::screenshot`] populated. The deterministic kittest path already attaches a + /// screenshot to every frame, so it treats this as a no-op. + Screenshot, + /// Toggle continuous screenshot mode. While `true`, the peer attaches a fresh + /// [`FrameScreenshot`] to every outgoing [`Frame`] until told otherwise. Useful for + /// inspectors that always want a current image (mirror the app's window) without + /// having to issue per-step [`Self::Screenshot`] requests. + /// + /// Kittest harnesses ignore this (they already screenshot every frame). + SetContinuousScreenshots(bool), + /// Resize the peer's viewport / harness to the given logical-point dimensions. + /// + /// Live apps issue a [`egui::ViewportCommand::InnerSize`]. The deterministic kittest + /// path calls `Harness::set_size`. + Resize { width: u32, height: u32 }, +} + +/// Hard cap on a single framed message. Matches the sanity limit enforced by both ends. +pub const MAX_MESSAGE_BYTES: usize = 256 * 1024 * 1024; // 256 MiB + +/// Read a length-prefixed MessagePack message. +/// +/// # Errors +/// I/O or decode failures. +pub fn read_message(mut reader: R) -> io::Result +where + R: Read, + T: for<'de> serde::Deserialize<'de>, +{ + let mut len_buf = [0u8; 4]; + reader.read_exact(&mut len_buf)?; + let len = u32::from_be_bytes(len_buf) as usize; + if len > MAX_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("message too large: {len} bytes"), + )); + } + let mut buf = vec![0u8; len]; + reader.read_exact(&mut buf)?; + rmp_serde::from_slice(&buf) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string())) +} + +/// Write a length-prefixed MessagePack message. +/// +/// # Errors +/// I/O or encode failures. +pub fn write_message(mut writer: W, value: &T) -> io::Result<()> +where + W: Write, + T: serde::Serialize, +{ + let bytes = rmp_serde::to_vec(value) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + let len = u32::try_from(bytes.len()) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + writer.write_all(&len.to_be_bytes())?; + writer.write_all(&bytes)?; + writer.flush()?; + Ok(()) +}