migrations/egui
1
0
Fork 0
mirror of https://github.com/emilk/egui.git synced 2026-09-02 06:40:06 -04:00
Code Activity

Add egui_inspection protocol and plugin (#8234)

Browse Source 
Introduces live inspection for running egui apps over a small TCP
request/response protocol, plus the `egui::Plugin` that serves it.

This is the minimal surface to get the egui mcp in, we may want to
extend this in the future to add support for the inspection gui.

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Lucas Meurer
2026-06-18 11:26:18 +02:00
committed by GitHub
parent 172fb54f7f
commit 86fcffb229
16 changed files with 869 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

364
crates/egui_inspection/src/plugin.rs Normal file
View File

@@ -0,0 +1,364 @@
//! [`InspectionPlugin`] — an [`egui::Plugin`] that lets an external inspector read the
//! AccessKit tree, inject input, and capture screenshots of a running app over a simple
//! request/response protocol ([`crate::protocol`]).
//!
//! # Model
//!
//! The plugin owns a list of in-flight requests. A connection thread (or a host with its own
//! transport) submits a [`Request`] through egui's own plugin
//! handle — `ctx.with_plugin::<InspectionPlugin, _>(|p| p.submit(req))` — which appends it and
//! returns a channel to await the single [`Response`] on, then calls `ctx.request_repaint()`
//! so an idle app wakes up to service it. The reply is produced on the UI thread inside the
//! plugin's hooks, which receive the [`egui::Context`] to issue repaints and viewport
//! commands — so the plugin never has to store a `Context` itself.
//!
//! [`serve`] binds a TCP listener; each accepted connection gets a thread that first writes
//! the protocol handshake, then loops reading framed [`Request`]s, submitting them, and
//! writing the framed [`Response`] back. Multiple clients are just multiple connections.
//!
//! Because egui locks each plugin only for the duration of a single hook call, a background
//! thread can take that same lock (via `with_plugin`) between hooks to enqueue work — so
//! egui's plugin handle *is* the cross-thread channel; no extra shared handle is needed.
//!
//! # Servicing
//!
//! Requests advance through a small per-request state machine across one or two frames:
//! `GetInfo` replies immediately; `GetTree` replies with the current frame's tree;
//! `Resize` / `ApplyEvents` apply their effect and reply [`Response::Done`] *after* the frame
//! has processed them (so a following `GetTree` reflects them); `GetScreenshot` dispatches a
//! viewport screenshot and replies once the resulting [`egui::Event::Screenshot`] arrives,
//! matched back to the request by a `user_data` id.
//!
//! Note that [`serve`]'s threads hold an [`egui::Context`] clone, so the context stays alive
//! for as long as the listener runs (the lifetime of the process, for a debug attach).
use std::sync::mpsc;
use std::time::Duration;
use egui::{Context, FullOutput, RawInput};
use crate::protocol::{EncodedPng, Request, Response};
/// How long [`serve`]'s connection threads wait for the UI thread before giving up. Generous:
/// a backgrounded window may not paint (and thus not service requests) for a while.
const REQUEST_TIMEOUT: Duration = Duration::from_secs(20);
/// Per-[`Request`] progress through the frame lifecycle.
#[derive(PartialEq, Eq)]
enum Phase {
/// Just submitted by a connection thread; not yet picked up by `input_hook`.
New,
/// Effect applied (or nothing to apply); reply at the end of this frame.
AwaitOutput,
/// A screenshot was dispatched with this `user_data` id; reply when the matching
/// [`egui::Event::Screenshot`] arrives.
AwaitScreenshot { id: u64 },
}
struct InFlight {
req: Request,
reply: mpsc::Sender<Response>,
phase: Phase,
}
/// An [`egui::Plugin`] that serves the inspection protocol. See the module docs.
pub struct InspectionPlugin {
/// Requests we haven't responded to yet.
in_flight: Vec<InFlight>,
step: u64,
/// Counter for screenshot `user_data` ids, so each [`egui::Event::Screenshot`] maps back
/// to the request that asked for it.
next_screenshot_id: u64,
/// App label reported in [`Response::Info`].
label: Option<String>,
}
impl InspectionPlugin {
/// Create the plugin and register it with [`Context::add_plugin`], then call [`serve`] to
/// listen on TCP (or feed it directly via `ctx.with_plugin(|p| p.submit(req))`).
pub fn new(label: Option<String>) -> Self {
Self {
in_flight: Vec::new(),
step: 0,
next_screenshot_id: 0,
label,
}
}
/// Submit a request; returns a channel that receives its single reply once the UI thread
/// services it. Call this through [`Context::with_plugin`] so it runs under egui's plugin
/// lock, then `request_repaint` and await the receiver *after* the lock is released.
pub fn submit(&mut self, req: Request) -> mpsc::Receiver<Response> {
let (tx, rx) = mpsc::channel();
self.in_flight.push(InFlight {
req,
reply: tx,
phase: Phase::New,
});
rx
}
/// While requests are still in flight, keep the UI loop spinning — reactive apps would
/// otherwise go idle between hooks before a screenshot round-trips.
fn maybe_repaint(&self, ctx: &Context) {
if !self.in_flight.is_empty() {
ctx.request_repaint();
}
}
}
impl egui::Plugin for InspectionPlugin {
fn debug_name(&self) -> &'static str {
"egui_inspection"
}
fn setup(&mut self, ctx: &Context) {
// The inspector describes the UI via the AccessKit tree.
ctx.enable_accesskit();
}
fn input_hook(&mut self, ctx: &Context, input: &mut RawInput) {
// Nothing in flight → idle frame, do no work.
if self.in_flight.is_empty() {
return;
}
// Match screenshot replies to the requests that asked for them, by `user_data` id. We
// observe (don't consume) the event so the host app still receives it.
for ev in &input.events {
let egui::Event::Screenshot {
user_data, image, ..
} = ev
else {
continue;
};
let Some(id) = user_data
.data
.as_ref()
.and_then(|d| d.downcast_ref::<u64>())
.copied()
else {
continue; // not one of ours
};
let png = match EncodedPng::from_color_image(image.as_ref()) {
Ok(png) => png,
Err(err) => {
// Shouldn't happen for a valid framebuffer; surface it loudly.
log::error!("egui_inspection: PNG encode failed: {err}");
continue;
}
};
self.in_flight.retain_mut(|item| {
if item.phase == (Phase::AwaitScreenshot { id }) {
let _ = item.reply.send(Response::Screenshot(png.clone()));
false
} else {
true
}
});
}
// Apply the input-side effect of new requests, dropping any that reply immediately.
// `label`/`next_id` are pulled out so the closure doesn't borrow `self` alongside the
// `retain_mut` borrow of `in_flight`.
let label = self.label.clone();
let mut next_id = self.next_screenshot_id;
self.in_flight.retain_mut(|item| {
if item.phase != Phase::New {
return true;
}
match &item.req {
Request::GetInfo => {
let _ = item.reply.send(Response::Info {
label: label.clone(),
egui_version: env!("CARGO_PKG_VERSION").to_owned(),
});
false
}
Request::GetTree => {
item.phase = Phase::AwaitOutput;
true
}
Request::ApplyEvents { events } => {
input.events.extend(events.iter().cloned());
// Reply with `Done` at the end of the frame so the agent can be sure the
// events were *executed* (e.g. a button click that created a file), not
// merely received.
item.phase = Phase::AwaitOutput;
true
}
Request::Resize { width, height } => {
ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(egui::vec2(
*width as f32,
*height as f32,
)));
item.phase = Phase::AwaitOutput;
true
}
Request::GetScreenshot => {
// Dispatch now so the command lands in this frame's output and the capture
// is one frame sooner; the pixels arrive in a later `input_hook`. The id
// ties that `Event::Screenshot` back to this request.
let id = next_id;
next_id += 1;
ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new(
id,
)));
item.phase = Phase::AwaitScreenshot { id };
true
}
}
});
self.next_screenshot_id = next_id;
self.maybe_repaint(ctx);
}
fn output_hook(&mut self, ctx: &Context, output: &mut FullOutput) {
self.step = self.step.saturating_add(1);
if self.in_flight.is_empty() {
return;
}
let step = self.step;
self.in_flight
.retain_mut(|item| match (&item.phase, &item.req) {
(Phase::AwaitOutput, Request::GetTree) => {
let _ = item.reply.send(Response::Tree {
step,
pixels_per_point: output.pixels_per_point,
accesskit: output.platform_output.accesskit_update.clone(),
});
false
}
(Phase::AwaitOutput, Request::ApplyEvents { .. } | Request::Resize { .. }) => {
let _ = item.reply.send(Response::Done);
false
}
_ => true,
});
self.maybe_repaint(ctx);
}
}
/// Attach inspection if enabled via the environment (see [`crate::bind_addr_from_env`]).
///
/// Registers an [`InspectionPlugin`] on `ctx` and starts serving on the configured address.
/// Returns `Ok(true)` when attached, `Ok(false)` when inspection is disabled.
///
/// # Errors
/// When the env-configured address can't be bound.
#[cfg(not(target_arch = "wasm32"))]
pub fn attach_from_env(ctx: &Context, label: Option<String>) -> std::io::Result<bool> {
let Some(addr) = crate::bind_addr_from_env() else {
return Ok(false);
};
ctx.add_plugin(InspectionPlugin::new(label));
serve(ctx, &addr)?;
Ok(true)
}
/// Bind a TCP listener at `addr` (e.g. `127.0.0.1:5719`) and accept inspector connections.
///
/// Drives the [`InspectionPlugin`] registered on `ctx`. Spawns one accept thread plus a
/// thread per connection (detached — they live for the process).
///
/// Binding a non-loopback address exposes the inspection port (and thus full control of the
/// app, plus its screenshots) to the network with no authentication — a warning is logged.
///
/// # Errors
/// When `addr` can't be parsed or bound.
#[cfg(not(target_arch = "wasm32"))]
pub fn serve(ctx: &Context, addr: &str) -> std::io::Result<()> {
use std::net::{TcpListener, ToSocketAddrs as _};
let resolved = addr
.to_socket_addrs()?
.next()
.ok_or_else(|| std::io::Error::other(format!("no address resolved from {addr:?}")))?;
let listener = TcpListener::bind(resolved)?;
let bound = listener.local_addr()?;
if bound.ip().is_loopback() {
log::info!("egui_inspection: listening on {bound}");
} else {
log::warn!(
"egui_inspection: listening on {bound} — the inspection port is reachable from \
the network with NO authentication; anyone who can reach it can drive the app \
and read its screen"
);
}
let ctx = ctx.clone();
std::thread::Builder::new()
.name("egui_inspection_accept".into())
.spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
let ctx = ctx.clone();
std::thread::Builder::new()
.name("egui_inspection_conn".into())
.spawn(move || {
if let Err(err) = serve_connection(stream, &ctx) {
log::warn!("egui_inspection: connection ended: {err}");
}
})
.expect("failed to spawn egui_inspection connection thread");
}
})?;
Ok(())
}
/// Connection handler: write the handshake, then read framed requests, submit each to the
/// plugin via the context, and write the framed response back. Returns once the client
/// disconnects.
///
/// # Errors
/// On any socket I/O failure.
#[cfg(not(target_arch = "wasm32"))]
fn serve_connection(stream: std::net::TcpStream, ctx: &Context) -> std::io::Result<()> {
use crate::protocol::{read_message, write_handshake, write_message};
let mut reader = std::io::BufReader::new(stream.try_clone()?);
let mut writer = std::io::BufWriter::new(stream);
// Identify ourselves and our protocol version before any framed messages.
write_handshake(&mut writer)?;
loop {
let req: Request = match read_message(&mut reader) {
Ok(req) => req,
Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), // client gone
Err(err) => return Err(err),
};
let Some(rx) = ctx.with_plugin::<InspectionPlugin, _>(|p| p.submit(req)) else {
return write_message(
&mut writer,
&Response::Error {
message: "egui_inspection plugin not registered".to_owned(),
},
);
};
// Wake the (possibly idle) UI loop so it services the request.
ctx.request_repaint();
let resp = rx.recv_timeout(REQUEST_TIMEOUT).unwrap_or_else(|_| {
// Almost always means the app isn't painting — e.g. the window is occluded or
// minimized, which on most platforms stops rendering. Surface it loudly.
log::error!(
"egui_inspection: request timed out after {REQUEST_TIMEOUT:?}; the app is not \
painting (is the window occluded or minimized?)"
);
Response::Error {
message: "request timed out — the app is not painting; bring its window to the \
foreground"
.to_owned(),
}
});
write_message(&mut writer, &resp)?;
}
}