diff --git a/Cargo.lock b/Cargo.lock index 76ed5b660..b003381a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,9 +450,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", @@ -1219,6 +1219,7 @@ dependencies = [ "egui-wgpu", "egui-winit", "egui_glow", + "egui_inspection", "glow", "glutin", "glutin-winit", @@ -1391,6 +1392,19 @@ dependencies = [ "winit", ] +[[package]] +name = "egui_inspection" +version = "0.34.3" +dependencies = [ + "document-features", + "egui", + "image", + "log", + "rmp-serde", + "serde", + "serde_bytes", +] + [[package]] name = "egui_kittest" version = "0.34.3" @@ -3947,6 +3961,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "ron" version = "0.12.0" @@ -4144,6 +4177,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index 1eb1d7737..1b1f2ba08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/egui_demo_lib", "crates/egui_extras", "crates/egui_glow", + "crates/egui_inspection", "crates/egui_kittest", "crates/egui-wgpu", "crates/egui-winit", @@ -65,6 +66,7 @@ egui_extras = { version = "0.34.3", path = "crates/egui_extras", default-feature egui-wgpu = { version = "0.34.3", path = "crates/egui-wgpu", default-features = false } egui_demo_lib = { version = "0.34.3", path = "crates/egui_demo_lib", default-features = false } egui_glow = { version = "0.34.3", path = "crates/egui_glow", default-features = false } +egui_inspection = { version = "0.34.3", path = "crates/egui_inspection", default-features = false } egui_kittest = { version = "0.34.3", path = "crates/egui_kittest", default-features = false } eframe = { version = "0.34.3", path = "crates/eframe", default-features = false } @@ -123,6 +125,7 @@ raw-window-handle = "0.6.2" rayon = "1.11.0" resvg = { version = "0.45.1", default-features = false } rfd = "0.17.2" +rmp-serde = "1.3.1" ron = "0.12.0" self_cell = "1.2.1" serde = { version = "1.0.228", features = ["derive"] } diff --git a/crates/eframe/Cargo.toml b/crates/eframe/Cargo.toml index 0530b64a8..3439610dc 100644 --- a/crates/eframe/Cargo.toml +++ b/crates/eframe/Cargo.toml @@ -116,6 +116,12 @@ x11 = [ ## This is used to generate images for examples. __screenshot = [] +## Enable the [`egui_inspection`] plugin. When the `EGUI_INSPECTION` (or +## `EGUI_INSPECTION_ADDR`) env var is set, eframe opens a TCP inspection port on startup that +## lets an external tool (e.g. the `egui_mcp` server) read the AccessKit tree, inject input, +## and capture screenshots. Off unless the env var is set; no-op on wasm. +inspection = ["dep:egui_inspection", "accesskit"] + [dependencies] egui = { workspace = true, default-features = false, features = ["bytemuck"] } @@ -131,6 +137,7 @@ web-time.workspace = true # Optional dependencies egui_glow = { workspace = true, optional = true, default-features = false } +egui_inspection = { workspace = true, optional = true, features = ["plugin"] } glow = { workspace = true, optional = true } ron = { workspace = true, optional = true, features = ["integer128"] } serde = { workspace = true, optional = true } diff --git a/crates/eframe/src/lib.rs b/crates/eframe/src/lib.rs index e5247a037..0e7259177 100644 --- a/crates/eframe/src/lib.rs +++ b/crates/eframe/src/lib.rs @@ -209,6 +209,35 @@ pub use native::file_storage::storage_dir; #[cfg(not(target_arch = "wasm32"))] pub mod icon_data; +// ---------------------------------------------------------------------------- + +/// Attach an [`egui_inspection::InspectionPlugin`] to `ctx` when enabled via the environment. +#[cfg(all(feature = "inspection", not(target_arch = "wasm32")))] +pub(crate) fn maybe_attach_inspection_plugin(ctx: &egui::Context, label: Option) { + match egui_inspection::attach_from_env(ctx, label) { + Ok(true) => log::info!("egui_inspection plugin attached"), + Ok(false) => {} + Err(err) => log::warn!("egui_inspection attach failed: {err}"), + } +} + +/// Fallback for native builds without the `inspection` feature. Logs warning if inspection env +/// var was set. +#[cfg(all( + not(feature = "inspection"), + not(target_arch = "wasm32"), + any(feature = "glow", feature = "wgpu_no_default_features") +))] +pub(crate) fn maybe_attach_inspection_plugin(_ctx: &egui::Context, _label: Option) { + if let Ok(value) = std::env::var("EGUI_INSPECTION") + && value != "0" + && value != "false" + && !value.is_empty() + { + log::warn!("Inspection env var set but app was compiled without eframe/inspection feature"); + } +} + /// This is how you start a native (desktop) app. /// /// The first argument is name of your app, which is an identifier diff --git a/crates/eframe/src/native/glow_integration.rs b/crates/eframe/src/native/glow_integration.rs index 2f10433c7..325070810 100644 --- a/crates/eframe/src/native/glow_integration.rs +++ b/crates/eframe/src/native/glow_integration.rs @@ -299,6 +299,9 @@ impl<'app> GlowWinitApp<'app> { let app_creator = std::mem::take(&mut self.app_creator) .expect("Single-use AppCreator has unexpectedly already been taken"); + #[cfg(all(not(feature = "inspection"), not(target_arch = "wasm32")))] + crate::maybe_attach_inspection_plugin(&integration.egui_ctx, Some(self.app_name.clone())); + let app: Box = { // Use latest raw_window_handle for eframe compatibility use raw_window_handle::{HasDisplayHandle as _, HasWindowHandle as _}; diff --git a/crates/eframe/src/native/wgpu_integration.rs b/crates/eframe/src/native/wgpu_integration.rs index a4da74d97..882cfb3b5 100644 --- a/crates/eframe/src/native/wgpu_integration.rs +++ b/crates/eframe/src/native/wgpu_integration.rs @@ -291,6 +291,9 @@ impl<'app> WgpuWinitApp<'app> { let app_creator = std::mem::take(&mut self.app_creator) .expect("Single-use AppCreator has unexpectedly already been taken"); + + crate::maybe_attach_inspection_plugin(&egui_ctx, Some(self.app_name.clone())); + let cc = CreationContext { egui_ctx: egui_ctx.clone(), integration_info: integration.frame.info().clone(), diff --git a/crates/egui/src/context.rs b/crates/egui/src/context.rs index 57e285575..e36448cf7 100644 --- a/crates/egui/src/context.rs +++ b/crates/egui/src/context.rs @@ -897,7 +897,7 @@ impl Context { profiling::function_scope!(); let plugins = self.read(|ctx| ctx.plugins.ordered_plugins()); - plugins.on_input(&mut new_input); + plugins.on_input(self, &mut new_input); self.write(|ctx| ctx.begin_pass(new_input)); } @@ -2391,7 +2391,7 @@ impl Context { let mut output = self.write(|ctx| ctx.end_pass()); let plugins = self.read(|ctx| ctx.plugins.ordered_plugins()); - plugins.on_output(&mut output); + plugins.on_output(self, &mut output); output } diff --git a/crates/egui/src/plugin.rs b/crates/egui/src/plugin.rs index 6bef04123..f8f1a4468 100644 --- a/crates/egui/src/plugin.rs +++ b/crates/egui/src/plugin.rs @@ -35,13 +35,13 @@ pub trait Plugin: Send + Sync + std::any::Any + 'static { /// /// Useful to inspect or modify the input. /// Since this is called outside a pass, don't show ui here. Using `Context::debug_painter` is fine though. - fn input_hook(&mut self, input: &mut RawInput) {} + fn input_hook(&mut self, ctx: &Context, input: &mut RawInput) {} /// Called just before the output is passed to the backend. /// /// Useful to inspect or modify the output. /// Since this is called outside a pass, don't show ui here. Using `Context::debug_painter` is fine though. - fn output_hook(&mut self, output: &mut FullOutput) {} + fn output_hook(&mut self, ctx: &Context, output: &mut FullOutput) {} /// Called when a widget is created and is under the pointer. /// @@ -168,17 +168,17 @@ impl PluginsOrdered { }); } - pub fn on_input(&self, input: &mut RawInput) { + pub fn on_input(&self, ctx: &Context, input: &mut RawInput) { profiling::scope!("plugins", "on_input"); self.for_each_dyn(|plugin| { - plugin.input_hook(input); + plugin.input_hook(ctx, input); }); } - pub fn on_output(&self, output: &mut FullOutput) { + pub fn on_output(&self, ctx: &Context, output: &mut FullOutput) { profiling::scope!("plugins", "on_output"); self.for_each_dyn(|plugin| { - plugin.output_hook(output); + plugin.output_hook(ctx, output); }); } diff --git a/crates/egui_demo_app/Cargo.toml b/crates/egui_demo_app/Cargo.toml index f9a153268..d024b9988 100644 --- a/crates/egui_demo_app/Cargo.toml +++ b/crates/egui_demo_app/Cargo.toml @@ -72,6 +72,9 @@ serde = { workspace = true, optional = true } # native: [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# Always enable eframe's `inspection` feature on native so `EGUI_INSPECTION=1 cargo run -p +# egui_demo_app` opens an inspection port for `egui_mcp` (no-op when the env var is unset). +eframe = { workspace = true, features = ["inspection"] } env_logger = { workspace = true, features = ["auto-color", "humantime"] } mimalloc.workspace = true rfd = { workspace = true, optional = true } diff --git a/crates/egui_demo_app/src/accessibility_inspector.rs b/crates/egui_demo_app/src/accessibility_inspector.rs index 95c72cc80..91da6c653 100644 --- a/crates/egui_demo_app/src/accessibility_inspector.rs +++ b/crates/egui_demo_app/src/accessibility_inspector.rs @@ -5,8 +5,8 @@ use accesskit_consumer::{FilterResult, Node, NodeId, Tree, TreeChangeHandler}; use eframe::epaint::text::TextWrapMode; use egui::{ - Button, Color32, Event, Frame, FullOutput, Id, Key, KeyboardShortcut, Label, Modifiers, Panel, - RawInput, RichText, ScrollArea, Ui, collapsing_header::CollapsingState, + Button, Color32, Context, Event, Frame, FullOutput, Id, Key, KeyboardShortcut, Label, + Modifiers, Panel, RawInput, RichText, ScrollArea, Ui, collapsing_header::CollapsingState, }; /// This [`egui::Plugin`] adds an inspector panel. @@ -46,7 +46,7 @@ impl egui::Plugin for AccessibilityInspectorPlugin { "Accessibility Inspector" } - fn input_hook(&mut self, input: &mut RawInput) { + fn input_hook(&mut self, _ctx: &Context, input: &mut RawInput) { if let Some(queued_action) = self.queued_action.take() { input .events @@ -54,7 +54,7 @@ impl egui::Plugin for AccessibilityInspectorPlugin { } } - fn output_hook(&mut self, output: &mut FullOutput) { + fn output_hook(&mut self, _ctx: &Context, output: &mut FullOutput) { if let Some(update) = output.platform_output.accesskit_update.clone() { self.tree = match mem::take(&mut self.tree) { None => { diff --git a/crates/egui_inspection/Cargo.toml b/crates/egui_inspection/Cargo.toml new file mode 100644 index 000000000..46f9caf9e --- /dev/null +++ b/crates/egui_inspection/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "egui_inspection" +version.workspace = true +authors = ["Lucas Meurer ", "Emil Ernerfeldt "] +description = "Protocol and egui::Plugin for inspection of egui apps" +edition.workspace = true +rust-version.workspace = true +homepage = "https://github.com/emilk/egui" +license.workspace = true +readme = "./README.md" +repository = "https://github.com/emilk/egui" +categories = ["gui", "development-tools::testing", "accessibility"] +keywords = ["gui", "egui", "inspector", "accesskit", "mcp"] +include = ["../../LICENSE-APACHE", "../../LICENSE-MIT", "**/*.rs", "Cargo.toml"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--generate-link-to-definition"] + +[features] +default = [] + +## Screenshot PNG encoding — the `EncodedPng::from_color_image` / `from_rgba` constructors. +png = ["dep:image", "image/png"] + +## `InspectionPlugin` — an `egui::Plugin` that serves the request/response inspection +## protocol over TCP. Apps usually enable inspection by setting the `EGUI_INSPECTION` env var +## (handled by eframe's `inspection` feature). +plugin = ["png", "dep:log"] + +[dependencies] +egui = { workspace = true, features = ["serde"] } +serde.workspace = true +serde_bytes = "0.11.17" +rmp-serde.workspace = true + +image = { workspace = true, optional = true } +log = { workspace = true, optional = true } +document-features = { workspace = true, optional = true } + +[lints] +workspace = true diff --git a/crates/egui_inspection/README.md b/crates/egui_inspection/README.md new file mode 100644 index 000000000..c6c80fd43 --- /dev/null +++ b/crates/egui_inspection/README.md @@ -0,0 +1,62 @@ +# egui_inspection + +[![Latest version](https://img.shields.io/crates/v/egui_inspection.svg)](https://crates.io/crates/egui_inspection) +[![Documentation](https://docs.rs/egui_inspection/badge.svg)](https://docs.rs/egui_inspection) + +Inspection for [egui](https://github.com/emilk/egui) apps. + +`egui_inspection` defines a wire protocol and an [`egui::Plugin`] (`InspectionPlugin`) that +serves it. An external inspector — such as the +[`egui_mcp`](https://crates.io/crates/egui_mcp) MCP server — connects and can: + +- read the app's **AccessKit tree** (`GetTree`), +- inject **input events** (`HandleEvents` — clicks, typing, scrolling, …), +- capture a **screenshot** on request (`Screenshot`), +- resize the window (`Resize`). + +The protocol is strictly request → response, which maps cleanly onto both a TCP socket and a +unary RPC (so the same machinery can be tunnelled over e.g. gRPC). + +> **Screenshots need a visible window.** Reading the tree and injecting input work even while +> the app is in the background, but capturing a screenshot requires a rendered frame — which +> the OS won't produce for a fully-occluded or minimized window (notably on macOS, where the +> GPU surface isn't available). Bring the window to the foreground to capture it; the +> `Screenshot` request times out otherwise. + +## What it's for + +`egui_inspection` is the shared foundation for tools that observe or drive an egui app from +the outside. Anything that speaks the protocol (over TCP, or another transport like gRPC) +can be a consumer: + +- **[`egui_mcp`](https://crates.io/crates/egui_mcp)** — an MCP server that exposes the app to + AI agents and other tooling: query the widget tree, click / type / scroll, take screenshots. +- **An egui inspector GUI** *(planned)* — a visual debugger that connects to a running app to + browse its widget tree and drive it interactively. +- **Test inspection & frame streaming** *(planned)* — attach to `egui_kittest` tests, and + stream frames for live mirroring of an app's window. + +## Enabling it in an eframe app + +Enable eframe's `inspection` feature, then set the `EGUI_INSPECTION` env var at runtime. It's +either truthy, falsy, or a bind address: + +```sh +EGUI_INSPECTION=1 cargo run --features inspection # binds 127.0.0.1:5719 +EGUI_INSPECTION=0.0.0.0:5719 cargo run --features inspection # reachable across devices +``` + +When the variable is unset or falsy (`0` / `false`), inspection is completely off +(production-safe). + +> ⚠️ Binding a non-loopback address exposes full control of the app — and its screenshots — +> to anyone who can reach the port, with **no authentication**. A warning is logged when you +> do so. Prefer loopback + an SSH tunnel for remote debugging. + +## Using the plugin directly + +```rust,no_run +# let ctx = egui::Context::default(); +ctx.add_plugin(egui_inspection::InspectionPlugin::new(Some("my app".to_owned()))); +egui_inspection::serve(&ctx, "127.0.0.1:5719").unwrap(); +``` diff --git a/crates/egui_inspection/src/lib.rs b/crates/egui_inspection/src/lib.rs new file mode 100644 index 000000000..c29eeff8e --- /dev/null +++ b/crates/egui_inspection/src/lib.rs @@ -0,0 +1,50 @@ +#![cfg_attr(doc, doc = include_str!("../README.md"))] +//! +//! ## Feature flags +#![cfg_attr(feature = "document-features", doc = document_features::document_features!())] + +pub mod protocol; + +pub use protocol::{ + EncodedPng, MAX_MESSAGE_BYTES, PROTOCOL_MAGIC, PROTOCOL_VERSION, Request, Response, + read_message, write_message, +}; + +/// The single environment variable that controls inspection. +/// +/// Interpreted by [`bind_addr_from_env`]: falsy (unset / empty / `0` / `false`) disables it; +/// truthy (`1` / `true`) enables it on [`DEFAULT_INSPECTION_ADDR`]; anything else is taken as +/// a `host:port` bind address (e.g. `0.0.0.0:5719` to expose it across the network). +pub const INSPECTION_ENV_VAR: &str = "EGUI_INSPECTION"; + +/// Default bind address used when [`INSPECTION_ENV_VAR`] is just truthy. +/// +/// Loopback only, on a fixed well-known port. The `egui_mcp` server defaults its `attach` to +/// this same port. +pub const DEFAULT_INSPECTION_ADDR: &str = "127.0.0.1:5719"; + +#[cfg(feature = "png")] +mod png; + +#[cfg(feature = "plugin")] +mod plugin; + +#[cfg(feature = "plugin")] +pub use plugin::InspectionPlugin; + +#[cfg(all(feature = "plugin", not(target_arch = "wasm32")))] +pub use plugin::{attach_from_env, serve}; + +/// Resolve the bind address from [`INSPECTION_ENV_VAR`], returning `None` when inspection is +/// disabled. Used by [`attach_from_env`] and eframe's auto-attach. +#[cfg(feature = "plugin")] +pub fn bind_addr_from_env() -> Option { + let value = std::env::var(INSPECTION_ENV_VAR).ok()?; + match value.trim() { + "" | "0" => None, + v if v.eq_ignore_ascii_case("false") => None, + "1" => Some(DEFAULT_INSPECTION_ADDR.to_owned()), + v if v.eq_ignore_ascii_case("true") => Some(DEFAULT_INSPECTION_ADDR.to_owned()), + addr => Some(addr.to_owned()), + } +} diff --git a/crates/egui_inspection/src/plugin.rs b/crates/egui_inspection/src/plugin.rs new file mode 100644 index 000000000..e3dafe573 --- /dev/null +++ b/crates/egui_inspection/src/plugin.rs @@ -0,0 +1,364 @@ +//! [`InspectionPlugin`] — an [`egui::Plugin`] that lets an external inspector read the +//! AccessKit tree, inject input, and capture screenshots of a running app over a simple +//! request/response protocol ([`crate::protocol`]). +//! +//! # Model +//! +//! The plugin owns a list of in-flight requests. A connection thread (or a host with its own +//! transport) submits a [`Request`] through egui's own plugin +//! handle — `ctx.with_plugin::(|p| p.submit(req))` — which appends it and +//! returns a channel to await the single [`Response`] on, then calls `ctx.request_repaint()` +//! so an idle app wakes up to service it. The reply is produced on the UI thread inside the +//! plugin's hooks, which receive the [`egui::Context`] to issue repaints and viewport +//! commands — so the plugin never has to store a `Context` itself. +//! +//! [`serve`] binds a TCP listener; each accepted connection gets a thread that first writes +//! the protocol handshake, then loops reading framed [`Request`]s, submitting them, and +//! writing the framed [`Response`] back. Multiple clients are just multiple connections. +//! +//! Because egui locks each plugin only for the duration of a single hook call, a background +//! thread can take that same lock (via `with_plugin`) between hooks to enqueue work — so +//! egui's plugin handle *is* the cross-thread channel; no extra shared handle is needed. +//! +//! # Servicing +//! +//! Requests advance through a small per-request state machine across one or two frames: +//! `GetInfo` replies immediately; `GetTree` replies with the current frame's tree; +//! `Resize` / `ApplyEvents` apply their effect and reply [`Response::Done`] *after* the frame +//! has processed them (so a following `GetTree` reflects them); `GetScreenshot` dispatches a +//! viewport screenshot and replies once the resulting [`egui::Event::Screenshot`] arrives, +//! matched back to the request by a `user_data` id. +//! +//! Note that [`serve`]'s threads hold an [`egui::Context`] clone, so the context stays alive +//! for as long as the listener runs (the lifetime of the process, for a debug attach). + +use std::sync::mpsc; +use std::time::Duration; + +use egui::{Context, FullOutput, RawInput}; + +use crate::protocol::{EncodedPng, Request, Response}; + +/// How long [`serve`]'s connection threads wait for the UI thread before giving up. Generous: +/// a backgrounded window may not paint (and thus not service requests) for a while. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); + +/// Per-[`Request`] progress through the frame lifecycle. +#[derive(PartialEq, Eq)] +enum Phase { + /// Just submitted by a connection thread; not yet picked up by `input_hook`. + New, + + /// Effect applied (or nothing to apply); reply at the end of this frame. + AwaitOutput, + + /// A screenshot was dispatched with this `user_data` id; reply when the matching + /// [`egui::Event::Screenshot`] arrives. + AwaitScreenshot { id: u64 }, +} + +struct InFlight { + req: Request, + reply: mpsc::Sender, + phase: Phase, +} + +/// An [`egui::Plugin`] that serves the inspection protocol. See the module docs. +pub struct InspectionPlugin { + /// Requests we haven't responded to yet. + in_flight: Vec, + + step: u64, + + /// Counter for screenshot `user_data` ids, so each [`egui::Event::Screenshot`] maps back + /// to the request that asked for it. + next_screenshot_id: u64, + + /// App label reported in [`Response::Info`]. + label: Option, +} + +impl InspectionPlugin { + /// Create the plugin and register it with [`Context::add_plugin`], then call [`serve`] to + /// listen on TCP (or feed it directly via `ctx.with_plugin(|p| p.submit(req))`). + pub fn new(label: Option) -> Self { + Self { + in_flight: Vec::new(), + step: 0, + next_screenshot_id: 0, + label, + } + } + + /// Submit a request; returns a channel that receives its single reply once the UI thread + /// services it. Call this through [`Context::with_plugin`] so it runs under egui's plugin + /// lock, then `request_repaint` and await the receiver *after* the lock is released. + pub fn submit(&mut self, req: Request) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(); + self.in_flight.push(InFlight { + req, + reply: tx, + phase: Phase::New, + }); + rx + } + + /// While requests are still in flight, keep the UI loop spinning — reactive apps would + /// otherwise go idle between hooks before a screenshot round-trips. + fn maybe_repaint(&self, ctx: &Context) { + if !self.in_flight.is_empty() { + ctx.request_repaint(); + } + } +} + +impl egui::Plugin for InspectionPlugin { + fn debug_name(&self) -> &'static str { + "egui_inspection" + } + + fn setup(&mut self, ctx: &Context) { + // The inspector describes the UI via the AccessKit tree. + ctx.enable_accesskit(); + } + + fn input_hook(&mut self, ctx: &Context, input: &mut RawInput) { + // Nothing in flight → idle frame, do no work. + if self.in_flight.is_empty() { + return; + } + + // Match screenshot replies to the requests that asked for them, by `user_data` id. We + // observe (don't consume) the event so the host app still receives it. + for ev in &input.events { + let egui::Event::Screenshot { + user_data, image, .. + } = ev + else { + continue; + }; + let Some(id) = user_data + .data + .as_ref() + .and_then(|d| d.downcast_ref::()) + .copied() + else { + continue; // not one of ours + }; + let png = match EncodedPng::from_color_image(image.as_ref()) { + Ok(png) => png, + Err(err) => { + // Shouldn't happen for a valid framebuffer; surface it loudly. + log::error!("egui_inspection: PNG encode failed: {err}"); + continue; + } + }; + self.in_flight.retain_mut(|item| { + if item.phase == (Phase::AwaitScreenshot { id }) { + let _ = item.reply.send(Response::Screenshot(png.clone())); + false + } else { + true + } + }); + } + + // Apply the input-side effect of new requests, dropping any that reply immediately. + // `label`/`next_id` are pulled out so the closure doesn't borrow `self` alongside the + // `retain_mut` borrow of `in_flight`. + let label = self.label.clone(); + let mut next_id = self.next_screenshot_id; + self.in_flight.retain_mut(|item| { + if item.phase != Phase::New { + return true; + } + match &item.req { + Request::GetInfo => { + let _ = item.reply.send(Response::Info { + label: label.clone(), + egui_version: env!("CARGO_PKG_VERSION").to_owned(), + }); + false + } + Request::GetTree => { + item.phase = Phase::AwaitOutput; + true + } + Request::ApplyEvents { events } => { + input.events.extend(events.iter().cloned()); + // Reply with `Done` at the end of the frame so the agent can be sure the + // events were *executed* (e.g. a button click that created a file), not + // merely received. + item.phase = Phase::AwaitOutput; + true + } + Request::Resize { width, height } => { + ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(egui::vec2( + *width as f32, + *height as f32, + ))); + item.phase = Phase::AwaitOutput; + true + } + Request::GetScreenshot => { + // Dispatch now so the command lands in this frame's output and the capture + // is one frame sooner; the pixels arrive in a later `input_hook`. The id + // ties that `Event::Screenshot` back to this request. + let id = next_id; + next_id += 1; + ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new( + id, + ))); + item.phase = Phase::AwaitScreenshot { id }; + true + } + } + }); + self.next_screenshot_id = next_id; + + self.maybe_repaint(ctx); + } + + fn output_hook(&mut self, ctx: &Context, output: &mut FullOutput) { + self.step = self.step.saturating_add(1); + if self.in_flight.is_empty() { + return; + } + + let step = self.step; + self.in_flight + .retain_mut(|item| match (&item.phase, &item.req) { + (Phase::AwaitOutput, Request::GetTree) => { + let _ = item.reply.send(Response::Tree { + step, + pixels_per_point: output.pixels_per_point, + accesskit: output.platform_output.accesskit_update.clone(), + }); + false + } + (Phase::AwaitOutput, Request::ApplyEvents { .. } | Request::Resize { .. }) => { + let _ = item.reply.send(Response::Done); + false + } + _ => true, + }); + + self.maybe_repaint(ctx); + } +} + +/// Attach inspection if enabled via the environment (see [`crate::bind_addr_from_env`]). +/// +/// Registers an [`InspectionPlugin`] on `ctx` and starts serving on the configured address. +/// Returns `Ok(true)` when attached, `Ok(false)` when inspection is disabled. +/// +/// # Errors +/// When the env-configured address can't be bound. +#[cfg(not(target_arch = "wasm32"))] +pub fn attach_from_env(ctx: &Context, label: Option) -> std::io::Result { + let Some(addr) = crate::bind_addr_from_env() else { + return Ok(false); + }; + ctx.add_plugin(InspectionPlugin::new(label)); + serve(ctx, &addr)?; + Ok(true) +} + +/// Bind a TCP listener at `addr` (e.g. `127.0.0.1:5719`) and accept inspector connections. +/// +/// Drives the [`InspectionPlugin`] registered on `ctx`. Spawns one accept thread plus a +/// thread per connection (detached — they live for the process). +/// +/// Binding a non-loopback address exposes the inspection port (and thus full control of the +/// app, plus its screenshots) to the network with no authentication — a warning is logged. +/// +/// # Errors +/// When `addr` can't be parsed or bound. +#[cfg(not(target_arch = "wasm32"))] +pub fn serve(ctx: &Context, addr: &str) -> std::io::Result<()> { + use std::net::{TcpListener, ToSocketAddrs as _}; + + let resolved = addr + .to_socket_addrs()? + .next() + .ok_or_else(|| std::io::Error::other(format!("no address resolved from {addr:?}")))?; + let listener = TcpListener::bind(resolved)?; + let bound = listener.local_addr()?; + if bound.ip().is_loopback() { + log::info!("egui_inspection: listening on {bound}"); + } else { + log::warn!( + "egui_inspection: listening on {bound} — the inspection port is reachable from \ + the network with NO authentication; anyone who can reach it can drive the app \ + and read its screen" + ); + } + + let ctx = ctx.clone(); + std::thread::Builder::new() + .name("egui_inspection_accept".into()) + .spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let ctx = ctx.clone(); + std::thread::Builder::new() + .name("egui_inspection_conn".into()) + .spawn(move || { + if let Err(err) = serve_connection(stream, &ctx) { + log::warn!("egui_inspection: connection ended: {err}"); + } + }) + .expect("failed to spawn egui_inspection connection thread"); + } + })?; + Ok(()) +} + +/// Connection handler: write the handshake, then read framed requests, submit each to the +/// plugin via the context, and write the framed response back. Returns once the client +/// disconnects. +/// +/// # Errors +/// On any socket I/O failure. +#[cfg(not(target_arch = "wasm32"))] +fn serve_connection(stream: std::net::TcpStream, ctx: &Context) -> std::io::Result<()> { + use crate::protocol::{read_message, write_handshake, write_message}; + + let mut reader = std::io::BufReader::new(stream.try_clone()?); + let mut writer = std::io::BufWriter::new(stream); + + // Identify ourselves and our protocol version before any framed messages. + write_handshake(&mut writer)?; + + loop { + let req: Request = match read_message(&mut reader) { + Ok(req) => req, + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), // client gone + Err(err) => return Err(err), + }; + let Some(rx) = ctx.with_plugin::(|p| p.submit(req)) else { + return write_message( + &mut writer, + &Response::Error { + message: "egui_inspection plugin not registered".to_owned(), + }, + ); + }; + // Wake the (possibly idle) UI loop so it services the request. + ctx.request_repaint(); + let resp = rx.recv_timeout(REQUEST_TIMEOUT).unwrap_or_else(|_| { + // Almost always means the app isn't painting — e.g. the window is occluded or + // minimized, which on most platforms stops rendering. Surface it loudly. + log::error!( + "egui_inspection: request timed out after {REQUEST_TIMEOUT:?}; the app is not \ + painting (is the window occluded or minimized?)" + ); + Response::Error { + message: "request timed out — the app is not painting; bring its window to the \ + foreground" + .to_owned(), + } + }); + write_message(&mut writer, &resp)?; + } +} diff --git a/crates/egui_inspection/src/png.rs b/crates/egui_inspection/src/png.rs new file mode 100644 index 000000000..1cb8d9381 --- /dev/null +++ b/crates/egui_inspection/src/png.rs @@ -0,0 +1,33 @@ +//! PNG encoding for screenshots — constructors for [`EncodedPng`]. + +use crate::protocol::EncodedPng; + +impl EncodedPng { + /// Encode an [`egui::ColorImage`] (e.g. from [`egui::Event::Screenshot`]) as PNG. + /// + /// # Errors + /// When the encoder fails. + pub fn from_color_image(image: &egui::ColorImage) -> Result { + let size = [image.size[0] as u32, image.size[1] as u32]; + Self::from_rgba(size, image.as_raw()) + } + + /// Encode tightly-packed RGBA8 pixels (`width * height * 4` bytes) as PNG. + /// + /// PNG keeps high-resolution captures off the hot path of socket throughput — a 1550×2114 + /// RGBA8 buffer is ~13 MiB raw but typically <1 MiB encoded. + /// + /// # Errors + /// When the encoder fails (e.g. the buffer length doesn't match `width * height * 4`). + pub fn from_rgba(size: [u32; 2], rgba: &[u8]) -> Result { + use image::ImageEncoder as _; + let mut bytes = Vec::new(); + image::codecs::png::PngEncoder::new(&mut bytes).write_image( + rgba, + size[0], + size[1], + image::ExtendedColorType::Rgba8, + )?; + Ok(Self { size, bytes }) + } +} diff --git a/crates/egui_inspection/src/protocol.rs b/crates/egui_inspection/src/protocol.rs new file mode 100644 index 000000000..c1810dd9a --- /dev/null +++ b/crates/egui_inspection/src/protocol.rs @@ -0,0 +1,213 @@ +//! Request/response wire protocol for inspecting a running egui app. +//! +//! Shared between an egui peer (a live `eframe` app running [`crate::InspectionPlugin`]) and +//! an external inspector (the `egui_mcp` server, or any other compatible tool). +//! +//! Every connection opens with a fixed binary handshake — [`PROTOCOL_MAGIC`] (4 bytes) plus +//! [`PROTOCOL_VERSION`] (4 big-endian bytes), written by the peer when a client connects — so +//! the client can reject a non-inspection or incompatible peer before decoding any +//! `MessagePack`. After the handshake the inspector sends [`Request`]s and the peer replies +//! with exactly one [`Response`] each. +//! +//! Messages are framed as a 4-byte big-endian length followed by a `MessagePack`-encoded body +//! (`rmp-serde`). Transport-neutral: the same framing works on TCP and any byte stream. +//! +//! Living in its own crate (rather than `egui_mcp`) lets eframe pull the protocol + plugin +//! in without depending on the MCP server, and lets external tools depend on the protocol +//! types directly. + +use std::io::{self, Read, Write}; + +use egui::accesskit; + +/// Wire-protocol version, sent in the connection handshake (see [`write_handshake`]). +/// +/// Bump on any non-additive change to [`Request`] / [`Response`]. +pub const PROTOCOL_VERSION: u32 = 1; + +/// Magic bytes that open every connection, identifying the egui inspection protocol. +pub const PROTOCOL_MAGIC: [u8; 4] = *b"eins"; + +/// Sent inspector → peer. The peer replies with exactly one [`Response`]. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum Request { + /// Read the peer's label. Reply: [`Response::Info`]. + GetInfo, + + /// Read the current AccessKit tree. Reply: [`Response::Tree`]. + /// + /// Servicing this triggers one repaint so the reply reflects the current frame. + GetTree, + + /// Capture the current framebuffer as PNG. Reply: [`Response::Screenshot`]. + /// + /// The peer issues an [`egui::ViewportCommand::Screenshot`] and replies once the + /// resulting [`egui::Event::Screenshot`] arrives (one extra frame). + GetScreenshot, + + /// Inject raw egui input events and run a frame. Reply: [`Response::Done`], returned only + /// *after* the events have been applied by a frame — so a subsequent [`Self::GetTree`] + /// observes their effect. This is the single channel for all interaction + /// (click / hover / drag / scroll / type / keypress): the inspector synthesizes the + /// appropriate [`egui::Event`]s and sends them here. + ApplyEvents { events: Vec }, + + /// Resize the peer's viewport to the given logical-point dimensions + /// (via [`egui::ViewportCommand::InnerSize`]). Reply: [`Response::Done`]. This is the one + /// action that isn't expressible as an [`egui::Event`]. + Resize { width: u32, height: u32 }, +} + +/// Sent peer → inspector, exactly one per [`Request`]. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum Response { + /// Reply to [`Request::GetInfo`]. + Info { + /// Human-readable identifier (app name), if the peer set one. + label: Option, + + /// egui version string (e.g. `"0.31.0"`). + egui_version: String, + }, + + /// Reply to [`Request::GetTree`]. + Tree { + /// Monotonically increasing frame counter. + step: u64, + + /// `physical_pixel = logical_point * pixels_per_point`. AccessKit bounds are in + /// logical coords; a screenshot is in physical pixels — multiply to align them. + pixels_per_point: f32, + + /// The current full AccessKit tree. egui rebuilds the complete node set every pass, + /// so this is a full snapshot, not an incremental update. `None` if AccessKit hasn't + /// produced a tree yet. + accesskit: Option, + }, + + /// Reply to [`Request::GetScreenshot`]. + Screenshot(EncodedPng), + + /// Reply to [`Request::ApplyEvents`] / [`Request::Resize`] — the action was *executed* + /// (not merely received): the events were processed by a frame, or the resize dispatched. + Done, + + /// The peer failed to service the request (recoverable; the connection stays open). + Error { message: String }, +} + +/// A PNG-encoded image with its pixel dimensions. +/// +/// Construct one with [`Self::from_color_image`] / [`Self::from_rgba`] (requires the `png` +/// feature). The data type itself is always available so the inspector side can carry it +/// without pulling in the encoder. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct EncodedPng { + /// `[width, height]` in physical pixels. + pub size: [u32; 2], + + /// PNG-encoded image bytes. `serde_bytes` encodes this as a msgpack `bin` blob (one type + /// tag + raw bytes) instead of the default per-byte `Vec` path. + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +/// Hard cap on a single framed message. Matches the sanity limit enforced by both ends. +pub const MAX_MESSAGE_BYTES: usize = 256 * 1024 * 1024; // 256 MiB + +fn invalid_data(err: impl std::fmt::Display) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, err.to_string()) +} + +/// Write the connection handshake: [`PROTOCOL_MAGIC`] followed by [`PROTOCOL_VERSION`] +/// (big-endian). The peer sends this first thing on every accepted connection. +/// +/// # Errors +/// On I/O failure. +pub fn write_handshake(mut writer: W) -> io::Result<()> { + writer.write_all(&PROTOCOL_MAGIC)?; + writer.write_all(&PROTOCOL_VERSION.to_be_bytes())?; + writer.flush() +} + +/// Read and validate the connection handshake, returning the peer's protocol version. +/// +/// # Errors +/// If the magic bytes don't match (not an egui inspection peer), or on I/O failure. +pub fn read_handshake(mut reader: R) -> io::Result { + let mut magic = [0u8; 4]; + reader.read_exact(&mut magic)?; + if magic != PROTOCOL_MAGIC { + return Err(invalid_data( + "not an egui_inspection peer (bad handshake magic)", + )); + } + let mut version = [0u8; 4]; + reader.read_exact(&mut version)?; + Ok(u32::from_be_bytes(version)) +} + +/// Encode a value into a length-prefixed `MessagePack` frame (4-byte big-endian length + body). +/// +/// This is the wire format; sync ([`read_message`]/[`write_message`]) and async transports +/// share it via these helpers so the framing stays in lockstep. +/// +/// # Errors +/// On encode failure or a body exceeding `u32::MAX`. +pub fn encode_frame(value: &T) -> io::Result> { + let body = rmp_serde::to_vec(value).map_err(invalid_data)?; + let len = u32::try_from(body.len()).map_err(invalid_data)?; + let mut frame = Vec::with_capacity(4 + body.len()); + frame.extend_from_slice(&len.to_be_bytes()); + frame.extend_from_slice(&body); + Ok(frame) +} + +/// Decode a 4-byte frame header into a body length, rejecting anything over [`MAX_MESSAGE_BYTES`]. +/// +/// # Errors +/// When the declared length exceeds the cap. +pub fn decode_frame_len(header: [u8; 4]) -> io::Result { + let len = u32::from_be_bytes(header) as usize; + if len > MAX_MESSAGE_BYTES { + return Err(invalid_data(format!("message too large: {len} bytes"))); + } + Ok(len) +} + +/// Decode a frame body (the bytes after the length prefix) into a value. +/// +/// # Errors +/// On decode failure. +pub fn decode_frame_body serde::Deserialize<'de>>(body: &[u8]) -> io::Result { + rmp_serde::from_slice(body).map_err(invalid_data) +} + +/// Read one length-prefixed `MessagePack` message. +/// +/// # Errors +/// I/O or decode failures. +pub fn read_message(mut reader: R) -> io::Result +where + R: Read, + T: for<'de> serde::Deserialize<'de>, +{ + let mut header = [0u8; 4]; + reader.read_exact(&mut header)?; + let mut body = vec![0u8; decode_frame_len(header)?]; + reader.read_exact(&mut body)?; + decode_frame_body(&body) +} + +/// Write one length-prefixed `MessagePack` message. +/// +/// # Errors +/// I/O or encode failures. +pub fn write_message(mut writer: W, value: &T) -> io::Result<()> +where + W: Write, + T: serde::Serialize, +{ + writer.write_all(&encode_frame(value)?)?; + writer.flush() +}