1
0
mirror of https://github.com/emilk/egui.git synced 2026-06-26 14:49:06 -04:00

Add egui_inspection protocol and plugin (#8234)

Introduces live inspection for running egui apps over a small TCP
request/response protocol, plus the `egui::Plugin` that serves it.

This is the minimal surface to get the egui mcp in, we may want to
extend this in the future to add support for the inspection gui.

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Lucas Meurer
2026-06-18 11:26:18 +02:00
committed by GitHub
parent 172fb54f7f
commit 86fcffb229
16 changed files with 869 additions and 14 deletions

View File

@@ -450,9 +450,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
[[package]]
name = "async-trait"
version = "0.1.83"
version = "0.1.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
@@ -1219,6 +1219,7 @@ dependencies = [
"egui-wgpu",
"egui-winit",
"egui_glow",
"egui_inspection",
"glow",
"glutin",
"glutin-winit",
@@ -1391,6 +1392,19 @@ dependencies = [
"winit",
]
[[package]]
name = "egui_inspection"
version = "0.34.3"
dependencies = [
"document-features",
"egui",
"image",
"log",
"rmp-serde",
"serde",
"serde_bytes",
]
[[package]]
name = "egui_kittest"
version = "0.34.3"
@@ -3947,6 +3961,25 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rmp"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
dependencies = [
"num-traits",
]
[[package]]
name = "rmp-serde"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"
dependencies = [
"rmp",
"serde",
]
[[package]]
name = "ron"
version = "0.12.0"
@@ -4144,6 +4177,16 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde_bytes"
version = "0.11.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
dependencies = [
"serde",
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.228"

View File

@@ -6,6 +6,7 @@ members = [
"crates/egui_demo_lib",
"crates/egui_extras",
"crates/egui_glow",
"crates/egui_inspection",
"crates/egui_kittest",
"crates/egui-wgpu",
"crates/egui-winit",
@@ -65,6 +66,7 @@ egui_extras = { version = "0.34.3", path = "crates/egui_extras", default-feature
egui-wgpu = { version = "0.34.3", path = "crates/egui-wgpu", default-features = false }
egui_demo_lib = { version = "0.34.3", path = "crates/egui_demo_lib", default-features = false }
egui_glow = { version = "0.34.3", path = "crates/egui_glow", default-features = false }
egui_inspection = { version = "0.34.3", path = "crates/egui_inspection", default-features = false }
egui_kittest = { version = "0.34.3", path = "crates/egui_kittest", default-features = false }
eframe = { version = "0.34.3", path = "crates/eframe", default-features = false }
@@ -123,6 +125,7 @@ raw-window-handle = "0.6.2"
rayon = "1.11.0"
resvg = { version = "0.45.1", default-features = false }
rfd = "0.17.2"
rmp-serde = "1.3.1"
ron = "0.12.0"
self_cell = "1.2.1"
serde = { version = "1.0.228", features = ["derive"] }

View File

@@ -116,6 +116,12 @@ x11 = [
## This is used to generate images for examples.
__screenshot = []
## Enable the [`egui_inspection`] plugin. When the `EGUI_INSPECTION` (or
## `EGUI_INSPECTION_ADDR`) env var is set, eframe opens a TCP inspection port on startup that
## lets an external tool (e.g. the `egui_mcp` server) read the AccessKit tree, inject input,
## and capture screenshots. Off unless the env var is set; no-op on wasm.
inspection = ["dep:egui_inspection", "accesskit"]
[dependencies]
egui = { workspace = true, default-features = false, features = ["bytemuck"] }
@@ -131,6 +137,7 @@ web-time.workspace = true
# Optional dependencies
egui_glow = { workspace = true, optional = true, default-features = false }
egui_inspection = { workspace = true, optional = true, features = ["plugin"] }
glow = { workspace = true, optional = true }
ron = { workspace = true, optional = true, features = ["integer128"] }
serde = { workspace = true, optional = true }

View File

@@ -209,6 +209,35 @@ pub use native::file_storage::storage_dir;
#[cfg(not(target_arch = "wasm32"))]
pub mod icon_data;
// ----------------------------------------------------------------------------
/// Attach an [`egui_inspection::InspectionPlugin`] to `ctx` when enabled via the environment.
#[cfg(all(feature = "inspection", not(target_arch = "wasm32")))]
pub(crate) fn maybe_attach_inspection_plugin(ctx: &egui::Context, label: Option<String>) {
match egui_inspection::attach_from_env(ctx, label) {
Ok(true) => log::info!("egui_inspection plugin attached"),
Ok(false) => {}
Err(err) => log::warn!("egui_inspection attach failed: {err}"),
}
}
/// Fallback for native builds without the `inspection` feature. Logs warning if inspection env
/// var was set.
#[cfg(all(
not(feature = "inspection"),
not(target_arch = "wasm32"),
any(feature = "glow", feature = "wgpu_no_default_features")
))]
pub(crate) fn maybe_attach_inspection_plugin(_ctx: &egui::Context, _label: Option<String>) {
if let Ok(value) = std::env::var("EGUI_INSPECTION")
&& value != "0"
&& value != "false"
&& !value.is_empty()
{
log::warn!("Inspection env var set but app was compiled without eframe/inspection feature");
}
}
/// This is how you start a native (desktop) app.
///
/// The first argument is name of your app, which is an identifier

View File

@@ -299,6 +299,9 @@ impl<'app> GlowWinitApp<'app> {
let app_creator = std::mem::take(&mut self.app_creator)
.expect("Single-use AppCreator has unexpectedly already been taken");
#[cfg(all(not(feature = "inspection"), not(target_arch = "wasm32")))]
crate::maybe_attach_inspection_plugin(&integration.egui_ctx, Some(self.app_name.clone()));
let app: Box<dyn 'app + App> = {
// Use latest raw_window_handle for eframe compatibility
use raw_window_handle::{HasDisplayHandle as _, HasWindowHandle as _};

View File

@@ -291,6 +291,9 @@ impl<'app> WgpuWinitApp<'app> {
let app_creator = std::mem::take(&mut self.app_creator)
.expect("Single-use AppCreator has unexpectedly already been taken");
crate::maybe_attach_inspection_plugin(&egui_ctx, Some(self.app_name.clone()));
let cc = CreationContext {
egui_ctx: egui_ctx.clone(),
integration_info: integration.frame.info().clone(),

View File

@@ -897,7 +897,7 @@ impl Context {
profiling::function_scope!();
let plugins = self.read(|ctx| ctx.plugins.ordered_plugins());
plugins.on_input(&mut new_input);
plugins.on_input(self, &mut new_input);
self.write(|ctx| ctx.begin_pass(new_input));
}
@@ -2391,7 +2391,7 @@ impl Context {
let mut output = self.write(|ctx| ctx.end_pass());
let plugins = self.read(|ctx| ctx.plugins.ordered_plugins());
plugins.on_output(&mut output);
plugins.on_output(self, &mut output);
output
}

View File

@@ -35,13 +35,13 @@ pub trait Plugin: Send + Sync + std::any::Any + 'static {
///
/// Useful to inspect or modify the input.
/// Since this is called outside a pass, don't show ui here. Using `Context::debug_painter` is fine though.
fn input_hook(&mut self, input: &mut RawInput) {}
fn input_hook(&mut self, ctx: &Context, input: &mut RawInput) {}
/// Called just before the output is passed to the backend.
///
/// Useful to inspect or modify the output.
/// Since this is called outside a pass, don't show ui here. Using `Context::debug_painter` is fine though.
fn output_hook(&mut self, output: &mut FullOutput) {}
fn output_hook(&mut self, ctx: &Context, output: &mut FullOutput) {}
/// Called when a widget is created and is under the pointer.
///
@@ -168,17 +168,17 @@ impl PluginsOrdered {
});
}
pub fn on_input(&self, input: &mut RawInput) {
pub fn on_input(&self, ctx: &Context, input: &mut RawInput) {
profiling::scope!("plugins", "on_input");
self.for_each_dyn(|plugin| {
plugin.input_hook(input);
plugin.input_hook(ctx, input);
});
}
pub fn on_output(&self, output: &mut FullOutput) {
pub fn on_output(&self, ctx: &Context, output: &mut FullOutput) {
profiling::scope!("plugins", "on_output");
self.for_each_dyn(|plugin| {
plugin.output_hook(output);
plugin.output_hook(ctx, output);
});
}

View File

@@ -72,6 +72,9 @@ serde = { workspace = true, optional = true }
# native:
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
# Always enable eframe's `inspection` feature on native so `EGUI_INSPECTION=1 cargo run -p
# egui_demo_app` opens an inspection port for `egui_mcp` (no-op when the env var is unset).
eframe = { workspace = true, features = ["inspection"] }
env_logger = { workspace = true, features = ["auto-color", "humantime"] }
mimalloc.workspace = true
rfd = { workspace = true, optional = true }

View File

@@ -5,8 +5,8 @@ use accesskit_consumer::{FilterResult, Node, NodeId, Tree, TreeChangeHandler};
use eframe::epaint::text::TextWrapMode;
use egui::{
Button, Color32, Event, Frame, FullOutput, Id, Key, KeyboardShortcut, Label, Modifiers, Panel,
RawInput, RichText, ScrollArea, Ui, collapsing_header::CollapsingState,
Button, Color32, Context, Event, Frame, FullOutput, Id, Key, KeyboardShortcut, Label,
Modifiers, Panel, RawInput, RichText, ScrollArea, Ui, collapsing_header::CollapsingState,
};
/// This [`egui::Plugin`] adds an inspector panel.
@@ -46,7 +46,7 @@ impl egui::Plugin for AccessibilityInspectorPlugin {
"Accessibility Inspector"
}
fn input_hook(&mut self, input: &mut RawInput) {
fn input_hook(&mut self, _ctx: &Context, input: &mut RawInput) {
if let Some(queued_action) = self.queued_action.take() {
input
.events
@@ -54,7 +54,7 @@ impl egui::Plugin for AccessibilityInspectorPlugin {
}
}
fn output_hook(&mut self, output: &mut FullOutput) {
fn output_hook(&mut self, _ctx: &Context, output: &mut FullOutput) {
if let Some(update) = output.platform_output.accesskit_update.clone() {
self.tree = match mem::take(&mut self.tree) {
None => {

View File

@@ -0,0 +1,42 @@
[package]
name = "egui_inspection"
version.workspace = true
authors = ["Lucas Meurer <hi@lucasmerlin.me>", "Emil Ernerfeldt <emil.ernerfeldt@gmail.com>"]
description = "Protocol and egui::Plugin for inspection of egui apps"
edition.workspace = true
rust-version.workspace = true
homepage = "https://github.com/emilk/egui"
license.workspace = true
readme = "./README.md"
repository = "https://github.com/emilk/egui"
categories = ["gui", "development-tools::testing", "accessibility"]
keywords = ["gui", "egui", "inspector", "accesskit", "mcp"]
include = ["../../LICENSE-APACHE", "../../LICENSE-MIT", "**/*.rs", "Cargo.toml"]
[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--generate-link-to-definition"]
[features]
default = []
## Screenshot PNG encoding — the `EncodedPng::from_color_image` / `from_rgba` constructors.
png = ["dep:image", "image/png"]
## `InspectionPlugin` — an `egui::Plugin` that serves the request/response inspection
## protocol over TCP. Apps usually enable inspection by setting the `EGUI_INSPECTION` env var
## (handled by eframe's `inspection` feature).
plugin = ["png", "dep:log"]
[dependencies]
egui = { workspace = true, features = ["serde"] }
serde.workspace = true
serde_bytes = "0.11.17"
rmp-serde.workspace = true
image = { workspace = true, optional = true }
log = { workspace = true, optional = true }
document-features = { workspace = true, optional = true }
[lints]
workspace = true

View File

@@ -0,0 +1,62 @@
# egui_inspection
[![Latest version](https://img.shields.io/crates/v/egui_inspection.svg)](https://crates.io/crates/egui_inspection)
[![Documentation](https://docs.rs/egui_inspection/badge.svg)](https://docs.rs/egui_inspection)
Inspection for [egui](https://github.com/emilk/egui) apps.
`egui_inspection` defines a wire protocol and an [`egui::Plugin`] (`InspectionPlugin`) that
serves it. An external inspector — such as the
[`egui_mcp`](https://crates.io/crates/egui_mcp) MCP server — connects and can:
- read the app's **AccessKit tree** (`GetTree`),
- inject **input events** (`HandleEvents` — clicks, typing, scrolling, …),
- capture a **screenshot** on request (`Screenshot`),
- resize the window (`Resize`).
The protocol is strictly request → response, which maps cleanly onto both a TCP socket and a
unary RPC (so the same machinery can be tunnelled over e.g. gRPC).
> **Screenshots need a visible window.** Reading the tree and injecting input work even while
> the app is in the background, but capturing a screenshot requires a rendered frame — which
> the OS won't produce for a fully-occluded or minimized window (notably on macOS, where the
> GPU surface isn't available). Bring the window to the foreground to capture it; the
> `Screenshot` request times out otherwise.
## What it's for
`egui_inspection` is the shared foundation for tools that observe or drive an egui app from
the outside. Anything that speaks the protocol (over TCP, or another transport like gRPC)
can be a consumer:
- **[`egui_mcp`](https://crates.io/crates/egui_mcp)** — an MCP server that exposes the app to
AI agents and other tooling: query the widget tree, click / type / scroll, take screenshots.
- **An egui inspector GUI** *(planned)* — a visual debugger that connects to a running app to
browse its widget tree and drive it interactively.
- **Test inspection & frame streaming** *(planned)* — attach to `egui_kittest` tests, and
stream frames for live mirroring of an app's window.
## Enabling it in an eframe app
Enable eframe's `inspection` feature, then set the `EGUI_INSPECTION` env var at runtime. It's
either truthy, falsy, or a bind address:
```sh
EGUI_INSPECTION=1 cargo run --features inspection # binds 127.0.0.1:5719
EGUI_INSPECTION=0.0.0.0:5719 cargo run --features inspection # reachable across devices
```
When the variable is unset or falsy (`0` / `false`), inspection is completely off
(production-safe).
> ⚠️ Binding a non-loopback address exposes full control of the app — and its screenshots —
> to anyone who can reach the port, with **no authentication**. A warning is logged when you
> do so. Prefer loopback + an SSH tunnel for remote debugging.
## Using the plugin directly
```rust,no_run
# let ctx = egui::Context::default();
ctx.add_plugin(egui_inspection::InspectionPlugin::new(Some("my app".to_owned())));
egui_inspection::serve(&ctx, "127.0.0.1:5719").unwrap();
```

View File

@@ -0,0 +1,50 @@
#![cfg_attr(doc, doc = include_str!("../README.md"))]
//!
//! ## Feature flags
#![cfg_attr(feature = "document-features", doc = document_features::document_features!())]
pub mod protocol;
pub use protocol::{
EncodedPng, MAX_MESSAGE_BYTES, PROTOCOL_MAGIC, PROTOCOL_VERSION, Request, Response,
read_message, write_message,
};
/// The single environment variable that controls inspection.
///
/// Interpreted by [`bind_addr_from_env`]: falsy (unset / empty / `0` / `false`) disables it;
/// truthy (`1` / `true`) enables it on [`DEFAULT_INSPECTION_ADDR`]; anything else is taken as
/// a `host:port` bind address (e.g. `0.0.0.0:5719` to expose it across the network).
pub const INSPECTION_ENV_VAR: &str = "EGUI_INSPECTION";
/// Default bind address used when [`INSPECTION_ENV_VAR`] is just truthy.
///
/// Loopback only, on a fixed well-known port. The `egui_mcp` server defaults its `attach` to
/// this same port.
pub const DEFAULT_INSPECTION_ADDR: &str = "127.0.0.1:5719";
#[cfg(feature = "png")]
mod png;
#[cfg(feature = "plugin")]
mod plugin;
#[cfg(feature = "plugin")]
pub use plugin::InspectionPlugin;
#[cfg(all(feature = "plugin", not(target_arch = "wasm32")))]
pub use plugin::{attach_from_env, serve};
/// Resolve the bind address from [`INSPECTION_ENV_VAR`], returning `None` when inspection is
/// disabled. Used by [`attach_from_env`] and eframe's auto-attach.
#[cfg(feature = "plugin")]
pub fn bind_addr_from_env() -> Option<String> {
let value = std::env::var(INSPECTION_ENV_VAR).ok()?;
match value.trim() {
"" | "0" => None,
v if v.eq_ignore_ascii_case("false") => None,
"1" => Some(DEFAULT_INSPECTION_ADDR.to_owned()),
v if v.eq_ignore_ascii_case("true") => Some(DEFAULT_INSPECTION_ADDR.to_owned()),
addr => Some(addr.to_owned()),
}
}

View File

@@ -0,0 +1,364 @@
//! [`InspectionPlugin`] — an [`egui::Plugin`] that lets an external inspector read the
//! AccessKit tree, inject input, and capture screenshots of a running app over a simple
//! request/response protocol ([`crate::protocol`]).
//!
//! # Model
//!
//! The plugin owns a list of in-flight requests. A connection thread (or a host with its own
//! transport) submits a [`Request`] through egui's own plugin
//! handle — `ctx.with_plugin::<InspectionPlugin, _>(|p| p.submit(req))` — which appends it and
//! returns a channel to await the single [`Response`] on, then calls `ctx.request_repaint()`
//! so an idle app wakes up to service it. The reply is produced on the UI thread inside the
//! plugin's hooks, which receive the [`egui::Context`] to issue repaints and viewport
//! commands — so the plugin never has to store a `Context` itself.
//!
//! [`serve`] binds a TCP listener; each accepted connection gets a thread that first writes
//! the protocol handshake, then loops reading framed [`Request`]s, submitting them, and
//! writing the framed [`Response`] back. Multiple clients are just multiple connections.
//!
//! Because egui locks each plugin only for the duration of a single hook call, a background
//! thread can take that same lock (via `with_plugin`) between hooks to enqueue work — so
//! egui's plugin handle *is* the cross-thread channel; no extra shared handle is needed.
//!
//! # Servicing
//!
//! Requests advance through a small per-request state machine across one or two frames:
//! `GetInfo` replies immediately; `GetTree` replies with the current frame's tree;
//! `Resize` / `ApplyEvents` apply their effect and reply [`Response::Done`] *after* the frame
//! has processed them (so a following `GetTree` reflects them); `GetScreenshot` dispatches a
//! viewport screenshot and replies once the resulting [`egui::Event::Screenshot`] arrives,
//! matched back to the request by a `user_data` id.
//!
//! Note that [`serve`]'s threads hold an [`egui::Context`] clone, so the context stays alive
//! for as long as the listener runs (the lifetime of the process, for a debug attach).
use std::sync::mpsc;
use std::time::Duration;
use egui::{Context, FullOutput, RawInput};
use crate::protocol::{EncodedPng, Request, Response};
/// How long [`serve`]'s connection threads wait for the UI thread before giving up. Generous:
/// a backgrounded window may not paint (and thus not service requests) for a while.
const REQUEST_TIMEOUT: Duration = Duration::from_secs(20);
/// Per-[`Request`] progress through the frame lifecycle.
#[derive(PartialEq, Eq)]
enum Phase {
/// Just submitted by a connection thread; not yet picked up by `input_hook`.
New,
/// Effect applied (or nothing to apply); reply at the end of this frame.
AwaitOutput,
/// A screenshot was dispatched with this `user_data` id; reply when the matching
/// [`egui::Event::Screenshot`] arrives.
AwaitScreenshot { id: u64 },
}
struct InFlight {
req: Request,
reply: mpsc::Sender<Response>,
phase: Phase,
}
/// An [`egui::Plugin`] that serves the inspection protocol. See the module docs.
pub struct InspectionPlugin {
/// Requests we haven't responded to yet.
in_flight: Vec<InFlight>,
step: u64,
/// Counter for screenshot `user_data` ids, so each [`egui::Event::Screenshot`] maps back
/// to the request that asked for it.
next_screenshot_id: u64,
/// App label reported in [`Response::Info`].
label: Option<String>,
}
impl InspectionPlugin {
/// Create the plugin and register it with [`Context::add_plugin`], then call [`serve`] to
/// listen on TCP (or feed it directly via `ctx.with_plugin(|p| p.submit(req))`).
pub fn new(label: Option<String>) -> Self {
Self {
in_flight: Vec::new(),
step: 0,
next_screenshot_id: 0,
label,
}
}
/// Submit a request; returns a channel that receives its single reply once the UI thread
/// services it. Call this through [`Context::with_plugin`] so it runs under egui's plugin
/// lock, then `request_repaint` and await the receiver *after* the lock is released.
pub fn submit(&mut self, req: Request) -> mpsc::Receiver<Response> {
let (tx, rx) = mpsc::channel();
self.in_flight.push(InFlight {
req,
reply: tx,
phase: Phase::New,
});
rx
}
/// While requests are still in flight, keep the UI loop spinning — reactive apps would
/// otherwise go idle between hooks before a screenshot round-trips.
fn maybe_repaint(&self, ctx: &Context) {
if !self.in_flight.is_empty() {
ctx.request_repaint();
}
}
}
impl egui::Plugin for InspectionPlugin {
fn debug_name(&self) -> &'static str {
"egui_inspection"
}
fn setup(&mut self, ctx: &Context) {
// The inspector describes the UI via the AccessKit tree.
ctx.enable_accesskit();
}
fn input_hook(&mut self, ctx: &Context, input: &mut RawInput) {
// Nothing in flight → idle frame, do no work.
if self.in_flight.is_empty() {
return;
}
// Match screenshot replies to the requests that asked for them, by `user_data` id. We
// observe (don't consume) the event so the host app still receives it.
for ev in &input.events {
let egui::Event::Screenshot {
user_data, image, ..
} = ev
else {
continue;
};
let Some(id) = user_data
.data
.as_ref()
.and_then(|d| d.downcast_ref::<u64>())
.copied()
else {
continue; // not one of ours
};
let png = match EncodedPng::from_color_image(image.as_ref()) {
Ok(png) => png,
Err(err) => {
// Shouldn't happen for a valid framebuffer; surface it loudly.
log::error!("egui_inspection: PNG encode failed: {err}");
continue;
}
};
self.in_flight.retain_mut(|item| {
if item.phase == (Phase::AwaitScreenshot { id }) {
let _ = item.reply.send(Response::Screenshot(png.clone()));
false
} else {
true
}
});
}
// Apply the input-side effect of new requests, dropping any that reply immediately.
// `label`/`next_id` are pulled out so the closure doesn't borrow `self` alongside the
// `retain_mut` borrow of `in_flight`.
let label = self.label.clone();
let mut next_id = self.next_screenshot_id;
self.in_flight.retain_mut(|item| {
if item.phase != Phase::New {
return true;
}
match &item.req {
Request::GetInfo => {
let _ = item.reply.send(Response::Info {
label: label.clone(),
egui_version: env!("CARGO_PKG_VERSION").to_owned(),
});
false
}
Request::GetTree => {
item.phase = Phase::AwaitOutput;
true
}
Request::ApplyEvents { events } => {
input.events.extend(events.iter().cloned());
// Reply with `Done` at the end of the frame so the agent can be sure the
// events were *executed* (e.g. a button click that created a file), not
// merely received.
item.phase = Phase::AwaitOutput;
true
}
Request::Resize { width, height } => {
ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(egui::vec2(
*width as f32,
*height as f32,
)));
item.phase = Phase::AwaitOutput;
true
}
Request::GetScreenshot => {
// Dispatch now so the command lands in this frame's output and the capture
// is one frame sooner; the pixels arrive in a later `input_hook`. The id
// ties that `Event::Screenshot` back to this request.
let id = next_id;
next_id += 1;
ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new(
id,
)));
item.phase = Phase::AwaitScreenshot { id };
true
}
}
});
self.next_screenshot_id = next_id;
self.maybe_repaint(ctx);
}
fn output_hook(&mut self, ctx: &Context, output: &mut FullOutput) {
self.step = self.step.saturating_add(1);
if self.in_flight.is_empty() {
return;
}
let step = self.step;
self.in_flight
.retain_mut(|item| match (&item.phase, &item.req) {
(Phase::AwaitOutput, Request::GetTree) => {
let _ = item.reply.send(Response::Tree {
step,
pixels_per_point: output.pixels_per_point,
accesskit: output.platform_output.accesskit_update.clone(),
});
false
}
(Phase::AwaitOutput, Request::ApplyEvents { .. } | Request::Resize { .. }) => {
let _ = item.reply.send(Response::Done);
false
}
_ => true,
});
self.maybe_repaint(ctx);
}
}
/// Attach inspection if enabled via the environment (see [`crate::bind_addr_from_env`]).
///
/// Registers an [`InspectionPlugin`] on `ctx` and starts serving on the configured address.
/// Returns `Ok(true)` when attached, `Ok(false)` when inspection is disabled.
///
/// # Errors
/// When the env-configured address can't be bound.
#[cfg(not(target_arch = "wasm32"))]
pub fn attach_from_env(ctx: &Context, label: Option<String>) -> std::io::Result<bool> {
let Some(addr) = crate::bind_addr_from_env() else {
return Ok(false);
};
ctx.add_plugin(InspectionPlugin::new(label));
serve(ctx, &addr)?;
Ok(true)
}
/// Bind a TCP listener at `addr` (e.g. `127.0.0.1:5719`) and accept inspector connections.
///
/// Drives the [`InspectionPlugin`] registered on `ctx`. Spawns one accept thread plus a
/// thread per connection (detached — they live for the process).
///
/// Binding a non-loopback address exposes the inspection port (and thus full control of the
/// app, plus its screenshots) to the network with no authentication — a warning is logged.
///
/// # Errors
/// When `addr` can't be parsed or bound.
#[cfg(not(target_arch = "wasm32"))]
pub fn serve(ctx: &Context, addr: &str) -> std::io::Result<()> {
use std::net::{TcpListener, ToSocketAddrs as _};
let resolved = addr
.to_socket_addrs()?
.next()
.ok_or_else(|| std::io::Error::other(format!("no address resolved from {addr:?}")))?;
let listener = TcpListener::bind(resolved)?;
let bound = listener.local_addr()?;
if bound.ip().is_loopback() {
log::info!("egui_inspection: listening on {bound}");
} else {
log::warn!(
"egui_inspection: listening on {bound} — the inspection port is reachable from \
the network with NO authentication; anyone who can reach it can drive the app \
and read its screen"
);
}
let ctx = ctx.clone();
std::thread::Builder::new()
.name("egui_inspection_accept".into())
.spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
let ctx = ctx.clone();
std::thread::Builder::new()
.name("egui_inspection_conn".into())
.spawn(move || {
if let Err(err) = serve_connection(stream, &ctx) {
log::warn!("egui_inspection: connection ended: {err}");
}
})
.expect("failed to spawn egui_inspection connection thread");
}
})?;
Ok(())
}
/// Connection handler: write the handshake, then read framed requests, submit each to the
/// plugin via the context, and write the framed response back. Returns once the client
/// disconnects.
///
/// # Errors
/// On any socket I/O failure.
#[cfg(not(target_arch = "wasm32"))]
fn serve_connection(stream: std::net::TcpStream, ctx: &Context) -> std::io::Result<()> {
use crate::protocol::{read_message, write_handshake, write_message};
let mut reader = std::io::BufReader::new(stream.try_clone()?);
let mut writer = std::io::BufWriter::new(stream);
// Identify ourselves and our protocol version before any framed messages.
write_handshake(&mut writer)?;
loop {
let req: Request = match read_message(&mut reader) {
Ok(req) => req,
Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), // client gone
Err(err) => return Err(err),
};
let Some(rx) = ctx.with_plugin::<InspectionPlugin, _>(|p| p.submit(req)) else {
return write_message(
&mut writer,
&Response::Error {
message: "egui_inspection plugin not registered".to_owned(),
},
);
};
// Wake the (possibly idle) UI loop so it services the request.
ctx.request_repaint();
let resp = rx.recv_timeout(REQUEST_TIMEOUT).unwrap_or_else(|_| {
// Almost always means the app isn't painting — e.g. the window is occluded or
// minimized, which on most platforms stops rendering. Surface it loudly.
log::error!(
"egui_inspection: request timed out after {REQUEST_TIMEOUT:?}; the app is not \
painting (is the window occluded or minimized?)"
);
Response::Error {
message: "request timed out — the app is not painting; bring its window to the \
foreground"
.to_owned(),
}
});
write_message(&mut writer, &resp)?;
}
}

View File

@@ -0,0 +1,33 @@
//! PNG encoding for screenshots — constructors for [`EncodedPng`].
use crate::protocol::EncodedPng;
impl EncodedPng {
/// Encode an [`egui::ColorImage`] (e.g. from [`egui::Event::Screenshot`]) as PNG.
///
/// # Errors
/// When the encoder fails.
pub fn from_color_image(image: &egui::ColorImage) -> Result<Self, image::ImageError> {
let size = [image.size[0] as u32, image.size[1] as u32];
Self::from_rgba(size, image.as_raw())
}
/// Encode tightly-packed RGBA8 pixels (`width * height * 4` bytes) as PNG.
///
/// PNG keeps high-resolution captures off the hot path of socket throughput — a 1550×2114
/// RGBA8 buffer is ~13 MiB raw but typically <1 MiB encoded.
///
/// # Errors
/// When the encoder fails (e.g. the buffer length doesn't match `width * height * 4`).
pub fn from_rgba(size: [u32; 2], rgba: &[u8]) -> Result<Self, image::ImageError> {
use image::ImageEncoder as _;
let mut bytes = Vec::new();
image::codecs::png::PngEncoder::new(&mut bytes).write_image(
rgba,
size[0],
size[1],
image::ExtendedColorType::Rgba8,
)?;
Ok(Self { size, bytes })
}
}

View File

@@ -0,0 +1,213 @@
//! Request/response wire protocol for inspecting a running egui app.
//!
//! Shared between an egui peer (a live `eframe` app running [`crate::InspectionPlugin`]) and
//! an external inspector (the `egui_mcp` server, or any other compatible tool).
//!
//! Every connection opens with a fixed binary handshake — [`PROTOCOL_MAGIC`] (4 bytes) plus
//! [`PROTOCOL_VERSION`] (4 big-endian bytes), written by the peer when a client connects — so
//! the client can reject a non-inspection or incompatible peer before decoding any
//! `MessagePack`. After the handshake the inspector sends [`Request`]s and the peer replies
//! with exactly one [`Response`] each.
//!
//! Messages are framed as a 4-byte big-endian length followed by a `MessagePack`-encoded body
//! (`rmp-serde`). Transport-neutral: the same framing works on TCP and any byte stream.
//!
//! Living in its own crate (rather than `egui_mcp`) lets eframe pull the protocol + plugin
//! in without depending on the MCP server, and lets external tools depend on the protocol
//! types directly.
use std::io::{self, Read, Write};
use egui::accesskit;
/// Wire-protocol version, sent in the connection handshake (see [`write_handshake`]).
///
/// Bump on any non-additive change to [`Request`] / [`Response`].
pub const PROTOCOL_VERSION: u32 = 1;
/// Magic bytes that open every connection, identifying the egui inspection protocol.
pub const PROTOCOL_MAGIC: [u8; 4] = *b"eins";
/// Sent inspector → peer. The peer replies with exactly one [`Response`].
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub enum Request {
/// Read the peer's label. Reply: [`Response::Info`].
GetInfo,
/// Read the current AccessKit tree. Reply: [`Response::Tree`].
///
/// Servicing this triggers one repaint so the reply reflects the current frame.
GetTree,
/// Capture the current framebuffer as PNG. Reply: [`Response::Screenshot`].
///
/// The peer issues an [`egui::ViewportCommand::Screenshot`] and replies once the
/// resulting [`egui::Event::Screenshot`] arrives (one extra frame).
GetScreenshot,
/// Inject raw egui input events and run a frame. Reply: [`Response::Done`], returned only
/// *after* the events have been applied by a frame — so a subsequent [`Self::GetTree`]
/// observes their effect. This is the single channel for all interaction
/// (click / hover / drag / scroll / type / keypress): the inspector synthesizes the
/// appropriate [`egui::Event`]s and sends them here.
ApplyEvents { events: Vec<egui::Event> },
/// Resize the peer's viewport to the given logical-point dimensions
/// (via [`egui::ViewportCommand::InnerSize`]). Reply: [`Response::Done`]. This is the one
/// action that isn't expressible as an [`egui::Event`].
Resize { width: u32, height: u32 },
}
/// Sent peer → inspector, exactly one per [`Request`].
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub enum Response {
/// Reply to [`Request::GetInfo`].
Info {
/// Human-readable identifier (app name), if the peer set one.
label: Option<String>,
/// egui version string (e.g. `"0.31.0"`).
egui_version: String,
},
/// Reply to [`Request::GetTree`].
Tree {
/// Monotonically increasing frame counter.
step: u64,
/// `physical_pixel = logical_point * pixels_per_point`. AccessKit bounds are in
/// logical coords; a screenshot is in physical pixels — multiply to align them.
pixels_per_point: f32,
/// The current full AccessKit tree. egui rebuilds the complete node set every pass,
/// so this is a full snapshot, not an incremental update. `None` if AccessKit hasn't
/// produced a tree yet.
accesskit: Option<accesskit::TreeUpdate>,
},
/// Reply to [`Request::GetScreenshot`].
Screenshot(EncodedPng),
/// Reply to [`Request::ApplyEvents`] / [`Request::Resize`] — the action was *executed*
/// (not merely received): the events were processed by a frame, or the resize dispatched.
Done,
/// The peer failed to service the request (recoverable; the connection stays open).
Error { message: String },
}
/// A PNG-encoded image with its pixel dimensions.
///
/// Construct one with [`Self::from_color_image`] / [`Self::from_rgba`] (requires the `png`
/// feature). The data type itself is always available so the inspector side can carry it
/// without pulling in the encoder.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct EncodedPng {
/// `[width, height]` in physical pixels.
pub size: [u32; 2],
/// PNG-encoded image bytes. `serde_bytes` encodes this as a msgpack `bin` blob (one type
/// tag + raw bytes) instead of the default per-byte `Vec<u8>` path.
#[serde(with = "serde_bytes")]
pub bytes: Vec<u8>,
}
/// Hard cap on a single framed message. Matches the sanity limit enforced by both ends.
pub const MAX_MESSAGE_BYTES: usize = 256 * 1024 * 1024; // 256 MiB
fn invalid_data(err: impl std::fmt::Display) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, err.to_string())
}
/// Write the connection handshake: [`PROTOCOL_MAGIC`] followed by [`PROTOCOL_VERSION`]
/// (big-endian). The peer sends this first thing on every accepted connection.
///
/// # Errors
/// On I/O failure.
pub fn write_handshake<W: Write>(mut writer: W) -> io::Result<()> {
writer.write_all(&PROTOCOL_MAGIC)?;
writer.write_all(&PROTOCOL_VERSION.to_be_bytes())?;
writer.flush()
}
/// Read and validate the connection handshake, returning the peer's protocol version.
///
/// # Errors
/// If the magic bytes don't match (not an egui inspection peer), or on I/O failure.
pub fn read_handshake<R: Read>(mut reader: R) -> io::Result<u32> {
let mut magic = [0u8; 4];
reader.read_exact(&mut magic)?;
if magic != PROTOCOL_MAGIC {
return Err(invalid_data(
"not an egui_inspection peer (bad handshake magic)",
));
}
let mut version = [0u8; 4];
reader.read_exact(&mut version)?;
Ok(u32::from_be_bytes(version))
}
/// Encode a value into a length-prefixed `MessagePack` frame (4-byte big-endian length + body).
///
/// This is the wire format; sync ([`read_message`]/[`write_message`]) and async transports
/// share it via these helpers so the framing stays in lockstep.
///
/// # Errors
/// On encode failure or a body exceeding `u32::MAX`.
pub fn encode_frame<T: serde::Serialize>(value: &T) -> io::Result<Vec<u8>> {
let body = rmp_serde::to_vec(value).map_err(invalid_data)?;
let len = u32::try_from(body.len()).map_err(invalid_data)?;
let mut frame = Vec::with_capacity(4 + body.len());
frame.extend_from_slice(&len.to_be_bytes());
frame.extend_from_slice(&body);
Ok(frame)
}
/// Decode a 4-byte frame header into a body length, rejecting anything over [`MAX_MESSAGE_BYTES`].
///
/// # Errors
/// When the declared length exceeds the cap.
pub fn decode_frame_len(header: [u8; 4]) -> io::Result<usize> {
let len = u32::from_be_bytes(header) as usize;
if len > MAX_MESSAGE_BYTES {
return Err(invalid_data(format!("message too large: {len} bytes")));
}
Ok(len)
}
/// Decode a frame body (the bytes after the length prefix) into a value.
///
/// # Errors
/// On decode failure.
pub fn decode_frame_body<T: for<'de> serde::Deserialize<'de>>(body: &[u8]) -> io::Result<T> {
rmp_serde::from_slice(body).map_err(invalid_data)
}
/// Read one length-prefixed `MessagePack` message.
///
/// # Errors
/// I/O or decode failures.
pub fn read_message<R, T>(mut reader: R) -> io::Result<T>
where
R: Read,
T: for<'de> serde::Deserialize<'de>,
{
let mut header = [0u8; 4];
reader.read_exact(&mut header)?;
let mut body = vec![0u8; decode_frame_len(header)?];
reader.read_exact(&mut body)?;
decode_frame_body(&body)
}
/// Write one length-prefixed `MessagePack` message.
///
/// # Errors
/// I/O or encode failures.
pub fn write_message<W, T>(mut writer: W, value: &T) -> io::Result<()>
where
W: Write,
T: serde::Serialize,
{
writer.write_all(&encode_frame(value)?)?;
writer.flush()
}