Move IME candidate window following text cursor (#258)

* egui_web: enable IME support on web.

* Move candidate window following text cursor.

* Preclude too frequent agent movement.

* IME candidate window move on native app.

egui_web/src/backend.rs

@@ -138,6 +138,7 @@ pub struct AppRunner {
     screen_reader: crate::screen_reader::ScreenReader,
     #[cfg(feature = "http")]
     http: Arc<http::WebHttp>,
+    pub(crate) text_cursor: Option<egui::Pos2>,
 }
 
 impl AppRunner {
@@ -156,6 +157,7 @@ impl AppRunner {
             screen_reader: Default::default(),
             #[cfg(feature = "http")]
             http: Arc::new(http::WebHttp {}),
+            text_cursor: None,
         })
     }
 
@@ -222,7 +224,7 @@ impl AppRunner {
         if self.web_backend.ctx.memory().options.screen_reader {
             self.screen_reader.speak(&egui_output.events_description());
         }
-        handle_output(&egui_output);
+        handle_output(&egui_output, self);
 
         {
             let epi::backend::AppOutput {

egui_web/src/lib.rs

@@ -28,7 +28,7 @@ use std::rc::Rc;
 use std::sync::Arc;
 use wasm_bindgen::prelude::*;
 
-static AGENT_ID: &str = "text_agent";
+static AGENT_ID: &str = "egui_text_agent";
 
 // ----------------------------------------------------------------------------
 // Helpers to hide some of the verbosity of web_sys
@@ -234,13 +234,14 @@ impl epi::Storage for LocalStorage {
 
 // ----------------------------------------------------------------------------
 
-pub fn handle_output(output: &egui::Output) {
+pub fn handle_output(output: &egui::Output, runner: &mut AppRunner) {
     let egui::Output {
         cursor_icon,
         open_url,
         copied_text,
         needs_repaint: _, // handled elsewhere
         events: _,        // we ignore these (TODO: accessibility screen reader)
+        text_cursor: cursor,
     } = output;
 
     set_cursor_icon(*cursor_icon);
@@ -255,6 +256,11 @@ pub fn handle_output(output: &egui::Output) {
 
     #[cfg(not(web_sys_unstable_apis))]
     let _ = copied_text;
+
+    if &runner.text_cursor != cursor {
+        move_text_cursor(cursor, runner.canvas_id());
+        runner.text_cursor = *cursor;
+    }
 }
 
 pub fn set_cursor_icon(cursor: egui::CursorIcon) -> Option<()> {
@@ -464,7 +470,7 @@ fn paint_and_schedule(runner_ref: AppRunnerRef) -> Result<(), JsValue> {
     request_animation_frame(runner_ref)
 }
 
-fn text_agent_hidden() -> bool {
+fn text_agent() -> web_sys::HtmlInputElement {
     use wasm_bindgen::JsCast;
     web_sys::window()
         .unwrap()
@@ -472,9 +478,8 @@ fn text_agent_hidden() -> bool {
         .unwrap()
         .get_element_by_id(AGENT_ID)
         .unwrap()
-        .dyn_into::<web_sys::HtmlInputElement>()
+        .dyn_into()
         .unwrap()
-        .hidden()
 }
 
 fn install_document_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
@@ -508,7 +513,7 @@ fn install_document_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
                 && !modifiers.command
                 && !should_ignore_key(&key)
                 // When text agent is shown, it sends text event instead.
-                && text_agent_hidden()
+                && text_agent().hidden()
             {
                 runner_lock.input.raw.events.push(egui::Event::Text(key));
             }
@@ -703,7 +708,6 @@ fn install_text_agent(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
         let input_clone = input.clone();
         let runner_ref = runner_ref.clone();
         let on_compositionend = Closure::wrap(Box::new(move |event: web_sys::CompositionEvent| {
-            // let event_type = event.type_();
             match event.type_().as_ref() {
                 "compositionstart" => {
                     is_composing.set(true);
@@ -995,3 +999,35 @@ fn manipulate_agent(canvas_id: &str, latest_cursor: Option<egui::Pos2>) -> Optio
     }
     Some(())
 }
+
+const MOBILE_DEVICE: [&str; 6] = ["Android", "iPhone", "iPad", "iPod", "webOS", "BlackBerry"];
+/// If context is running under mobile device?
+fn is_mobile() -> Option<bool> {
+    let user_agent = web_sys::window()?.navigator().user_agent().ok()?;
+    let is_mobile = MOBILE_DEVICE.iter().any(|&name| user_agent.contains(name));
+    Some(is_mobile)
+}
+
+// Move angnt to text cursor's position, on desktop/laptop, candidate window moves following text elemt(agent),
+// so it appears that the IME candidate window moves with text cursor.
+// On mobile devices, there is no need to do that.
+fn move_text_cursor(cursor: &Option<egui::Pos2>, canvas_id: &str) -> Option<()> {
+    let style = text_agent().style();
+    // Note: movint agent on mobile devices will lead to unpreditable scroll.
+    if is_mobile() == Some(false) {
+        cursor.as_ref().and_then(|&egui::Pos2 { x, y }| {
+            let canvas = canvas_element(canvas_id)?;
+            let y = y + (canvas.scroll_top() + canvas.offset_top()) as f32;
+            let x = x + (canvas.scroll_left() + canvas.offset_left()) as f32;
+            // Canvas is translated 50% horizontally in html.
+            let x = x - canvas.offset_width() as f32 / 2.0;
+            style.set_property("position", "absolute").ok()?;
+            style.set_property("top", &(y.to_string() + "px")).ok()?;
+            style.set_property("left", &(x.to_string() + "px")).ok()
+        })
+    } else {
+        style.set_property("position", "absolute").ok()?;
+        style.set_property("top", "0px").ok()?;
+        style.set_property("left", "0px").ok()
+    }
+}