From e1e6442ff5f03c7a7a7dbe290ca4f6f84c755c78 Mon Sep 17 00:00:00 2001 From: Butterfly Dev Date: Tue, 7 Apr 2026 04:36:29 +0000 Subject: [PATCH] =?UTF-8?q?agent:=20input.rs=20=E2=80=94=20full=20remote?= =?UTF-8?q?=20control:=20mouse=20move/click/dblclick/scroll,=20keyboard=20?= =?UTF-8?q?with=2060+=20key=20mappings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/src/input.rs | 406 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 agent/src/input.rs diff --git a/agent/src/input.rs b/agent/src/input.rs new file mode 100644 index 0000000..b57c350 --- /dev/null +++ b/agent/src/input.rs @@ -0,0 +1,406 @@ +//! Input simulation module. +//! +//! Receives HUD commands from the Butterfly server (forwarded from browser viewers) +//! and executes them as real mouse/keyboard events on the local machine. This is +//! the core of the remote control functionality — a viewer in the browser can +//! move the mouse, click, type, and scroll on the remote machine in real time. +//! +//! Uses the `enigo` crate for cross-platform input injection: +//! - Windows: SendInput Win32 API +//! - macOS: CoreGraphics CGEvent +//! - Linux: X11 XTest extension + +use anyhow::Result; +use enigo::{Button, Direction, Enigo, Key, Settings, Coordinate}; +use log::{info, warn}; + +/// Manages input simulation on the local machine. +pub struct InputHandler { + enigo: Enigo, + screen_width: u32, + screen_height: u32, +} + +impl InputHandler { + /// Create a new input handler. + /// + /// `screen_width` and `screen_height` are the dimensions of the captured + /// display. Mouse coordinates from viewers are in this coordinate space. + pub fn new(screen_width: u32, screen_height: u32) -> Result { + let enigo = Enigo::new(&Settings::default()) + .map_err(|e| anyhow::anyhow!("failed to initialize input handler: {}", e))?; + + info!( + "input handler initialized (screen {}x{})", + screen_width, screen_height + ); + + Ok(Self { + enigo, + screen_width, + screen_height, + }) + } + + /// Execute a HUD command received from a viewer. + /// + /// Commands: + /// - `mouse_move` — Move the cursor to (x, y). + /// - `mouse_down` — Press a mouse button (left/middle/right). + /// - `mouse_up` — Release a mouse button. + /// - `mouse_click` — Click a mouse button (press + release). + /// - `mouse_dblclick` — Double-click a mouse button. + /// - `scroll` — Scroll vertically/horizontally. + /// - `key_down` — Press a key. + /// - `key_up` — Release a key. + /// - `key_click` — Type a key (press + release). + /// - `key_type` — Type a string of characters. + pub fn execute(&mut self, command: &str, params: &serde_json::Value) -> Result<()> { + match command { + "mouse_move" => self.mouse_move(params), + "mouse_down" => self.mouse_down(params), + "mouse_up" => self.mouse_up(params), + "mouse_click" => self.mouse_click(params), + "mouse_dblclick" => self.mouse_double_click(params), + "scroll" => self.scroll(params), + "key_down" => self.key_down(params), + "key_up" => self.key_up(params), + "key_click" => self.key_click(params), + "key_type" => self.key_type(params), + _ => { + warn!("unknown HUD command: {}", command); + Ok(()) + } + } + } + + // ── Mouse commands ───────────────────────────────────────────────────── + + /// Move the mouse cursor to the specified position. + /// + /// Params: `{ "x": number, "y": number }` + fn mouse_move(&mut self, params: &serde_json::Value) -> Result<()> { + let x = params["x"].as_i64().unwrap_or(0) as i32; + let y = params["y"].as_i64().unwrap_or(0) as i32; + + // Clamp to screen bounds. + let x = x.clamp(0, self.screen_width as i32 - 1); + let y = y.clamp(0, self.screen_height as i32 - 1); + + self.enigo + .move_mouse(x, y, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("mouse_move failed: {}", e))?; + + Ok(()) + } + + /// Press a mouse button down. + /// + /// Params: `{ "button": number }` — 0=left, 1=middle, 2=right + fn mouse_down(&mut self, params: &serde_json::Value) -> Result<()> { + let button = json_to_button(params["button"].as_i64().unwrap_or(0)); + self.enigo + .button(button, Direction::Press) + .map_err(|e| anyhow::anyhow!("mouse_down failed: {}", e))?; + Ok(()) + } + + /// Release a mouse button. + fn mouse_up(&mut self, params: &serde_json::Value) -> Result<()> { + let button = json_to_button(params["button"].as_i64().unwrap_or(0)); + self.enigo + .button(button, Direction::Release) + .map_err(|e| anyhow::anyhow!("mouse_up failed: {}", e))?; + Ok(()) + } + + /// Click a mouse button (press + release). + fn mouse_click(&mut self, params: &serde_json::Value) -> Result<()> { + let button = json_to_button(params["button"].as_i64().unwrap_or(0)); + self.enigo + .button(button, Direction::Click) + .map_err(|e| anyhow::anyhow!("mouse_click failed: {}", e))?; + Ok(()) + } + + /// Double-click a mouse button. + fn mouse_double_click(&mut self, params: &serde_json::Value) -> Result<()> { + let button = json_to_button(params["button"].as_i64().unwrap_or(0)); + // Double-click = two rapid clicks. + self.enigo + .button(button, Direction::Click) + .map_err(|e| anyhow::anyhow!("mouse_dblclick (1st) failed: {}", e))?; + std::thread::sleep(std::time::Duration::from_millis(30)); + self.enigo + .button(button, Direction::Click) + .map_err(|e| anyhow::anyhow!("mouse_dblclick (2nd) failed: {}", e))?; + Ok(()) + } + + /// Scroll the mouse wheel. + /// + /// Params: `{ "deltaX": number, "deltaY": number }` + /// Positive deltaY = scroll down, negative = scroll up. + fn scroll(&mut self, params: &serde_json::Value) -> Result<()> { + let delta_y = params["deltaY"].as_i64().unwrap_or(0) as i32; + let delta_x = params["deltaX"].as_i64().unwrap_or(0) as i32; + + // Vertical scroll. + if delta_y != 0 { + let direction = if delta_y > 0 { + Direction::ScrollDown + } else { + Direction::ScrollUp + }; + let amount = delta_y.abs().min(50); // Clamp to prevent crazy scrolling. + for _ in 0..amount { + self.enigo + .scroll_y(1, direction) + .map_err(|e| anyhow::anyhow!("scroll_y failed: {}", e))?; + } + } + + // Horizontal scroll. + if delta_x != 0 { + let direction = if delta_x > 0 { + Direction::ScrollRight + } else { + Direction::ScrollLeft + }; + let amount = delta_x.abs().min(50); + for _ in 0..amount { + self.enigo + .scroll_x(1, direction) + .map_err(|e| anyhow::anyhow!("scroll_x failed: {}", e))?; + } + } + + Ok(()) + } + + // ── Keyboard commands ────────────────────────────────────────────────── + + /// Press a key down. + /// + /// Params: `{ "key": string, "code": string, "ctrl": bool, "shift": bool, "alt": bool, "meta": bool }` + fn key_down(&mut self, params: &serde_json::Value) -> Result<()> { + let key = map_key( + params["code"].as_str().unwrap_or(""), + params["key"].as_str().unwrap_or(""), + ); + + // Hold modifiers first (order matters for some combinations). + if params["ctrl"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Control, Direction::Press); + } + if params["shift"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Shift, Direction::Press); + } + if params["alt"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Alt, Direction::Press); + } + if params["meta"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Meta, Direction::Press); + } + + self.enigo + .key(key, Direction::Press) + .map_err(|e| anyhow::anyhow!("key_down failed: {}", e))?; + + Ok(()) + } + + /// Release a key. + fn key_up(&mut self, params: &serde_json::Value) -> Result<()> { + let key = map_key( + params["code"].as_str().unwrap_or(""), + params["key"].as_str().unwrap_or(""), + ); + + // Release the main key first. + let _ = self.enigo.key(key, Direction::Release); + + // Release modifiers in reverse order. + if params["meta"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Meta, Direction::Release); + } + if params["alt"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Alt, Direction::Release); + } + if params["shift"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Shift, Direction::Release); + } + if params["ctrl"].as_bool().unwrap_or(false) { + let _ = self.enigo.key(Key::Control, Direction::Release); + } + + Ok(()) + } + + /// Type a single key (press + release). + fn key_click(&mut self, params: &serde_json::Value) -> Result<()> { + let key = map_key( + params["code"].as_str().unwrap_or(""), + params["key"].as_str().unwrap_or(""), + ); + + self.enigo + .key(key, Direction::Click) + .map_err(|e| anyhow::anyhow!("key_click failed: {}", e))?; + + Ok(()) + } + + /// Type a string of characters. + /// + /// Params: `{ "text": string }` + fn key_type(&mut self, params: &serde_json::Value) -> Result<()> { + let text = params["text"].as_str().unwrap_or(""); + for ch in text.chars() { + // Skip control characters. + if ch.is_control() { + continue; + } + if let Err(e) = self.enigo.key(Key::Unicode(ch), Direction::Click) { + warn!("key_type failed for '{}': {}", ch, e); + } + } + Ok(()) + } +} + +// ── Helper functions ────────────────────────────────────────────────────────── + +/// Convert a numeric button index to an `enigo::Button`. +/// +/// 0 = Left, 1 = Middle, 2 = Right, 3 = Back, 4 = Forward +fn json_to_button(idx: i64) -> Button { + match idx { + 1 => Button::Middle, + 2 => Button::Right, + 3 => Button::Back, + 4 => Button::Forward, + _ => Button::Left, + } +} + +/// Map a browser keyboard event to an `enigo::Key`. +/// +/// `code` is the physical key position (e.g., "KeyA", "ShiftLeft", "Digit1"). +/// `key` is the logical key value (e.g., "a", "Shift", "1"). +/// +/// Strategy: +/// 1. If `key` is a single printable character, use `Key::Unicode(char)`. +/// 2. Otherwise, map `code` to the corresponding `Key` variant. +fn map_key(code: &str, key: &str) -> Key { + // Printable single character — use Unicode for international keyboard support. + if key.chars().count() == 1 { + let ch = key.chars().next().unwrap(); + if !ch.is_control() { + return Key::Unicode(ch); + } + } + + // Map physical key codes to enigo Key variants. + match code { + // ── Modifier keys ────────────────────────────────────────────── + "ShiftLeft" | "ShiftRight" => Key::Shift, + "ControlLeft" | "ControlRight" => Key::Control, + "AltLeft" | "AltRight" => Key::Alt, + "MetaLeft" | "MetaRight" => Key::Meta, + + // ── Whitespace / editing ──────────────────────────────────────── + "Backspace" => Key::Backspace, + "Tab" => Key::Tab, + "Enter" => Key::Return, + "Space" => Key::Space, + "Escape" => Key::Escape, + + // ── Arrow keys ────────────────────────────────────────────────── + "ArrowUp" => Key::Up, + "ArrowDown" => Key::Down, + "ArrowLeft" => Key::Left, + "ArrowRight" => Key::Right, + + // ── Function keys ─────────────────────────────────────────────── + "F1" => Key::F1, + "F2" => Key::F2, + "F3" => Key::F3, + "F4" => Key::F4, + "F5" => Key::F5, + "F6" => Key::F6, + "F7" => Key::F7, + "F8" => Key::F8, + "F9" => Key::F9, + "F10" => Key::F10, + "F11" => Key::F11, + "F12" => Key::F12, + "F13" => Key::F13, + "F14" => Key::F14, + "F15" => Key::F15, + "F16" => Key::F16, + "F17" => Key::F17, + "F18" => Key::F18, + "F19" => Key::F19, + "F20" => Key::F20, + + // ── Lock keys ─────────────────────────────────────────────────── + "CapsLock" => Key::CapsLock, + "NumLock" => Key::NumLock, + "ScrollLock" => Key::ScrollLock, + + // ── Navigation / editing cluster ──────────────────────────────── + "Insert" => Key::Insert, + "Delete" => Key::Delete, + "Home" => Key::Home, + "End" => Key::End, + "PageUp" => Key::PageUp, + "PageDown" => Key::PageDown, + "PrintScreen" => Key::PrintScreen, + "Pause" => Key::Pause, + + // ── Numpad ────────────────────────────────────────────────────── + "Numpad0" => Key::Numpad0, + "Numpad1" => Key::Numpad1, + "Numpad2" => Key::Numpad2, + "Numpad3" => Key::Numpad3, + "Numpad4" => Key::Numpad4, + "Numpad5" => Key::Numpad5, + "Numpad6" => Key::Numpad6, + "Numpad7" => Key::Numpad7, + "Numpad8" => Key::Numpad8, + "Numpad9" => Key::Numpad9, + "NumpadAdd" => Key::NumpadAdd, + "NumpadSubtract" => Key::NumpadSubtract, + "NumpadMultiply" => Key::NumpadMultiply, + "NumpadDivide" => Key::NumpadDivide, + "NumpadDecimal" => Key::NumpadDecimal, + "NumpadEnter" => Key::NumpadEnter, + "NumLock" => Key::NumLock, + + // ── Punctuation / symbols ─────────────────────────────────────── + // These are handled as Unicode characters when sent via `key` field. + // If only `code` is available, fall back to Unicode mapping. + "BracketLeft" => Key::Unicode('['), + "BracketRight" => Key::Unicode(']'), + "Semicolon" => Key::Unicode(';'), + "Quote" => Key::Unicode('\''), + "Backquote" => Key::Unicode('`'), + "Backslash" => Key::Unicode('\\'), + "Slash" => Key::Unicode('/'), + "Comma" => Key::Unicode(','), + "Period" => Key::Unicode('.'), + "Minus" => Key::Unicode('-'), + "Equal" => Key::Unicode('='), + + // ── Fallback: try to use the key value as Unicode ─────────────── + _ => { + if !key.is_empty() && key.chars().count() == 1 { + Key::Unicode(key.chars().next().unwrap()) + } else { + warn!("unmapped key code '{}', key '{}'", code, key); + Key::Unicode('\0') // No-op key + } + } + } +}