//! Input simulation module. //! //! Receives HUD commands from the Butterfly server (forwarded from browser viewers) //! and executes them as real mouse/keyboard events on the local machine. This is //! the core of the remote control functionality — a viewer in the browser can //! move the mouse, click, type, and scroll on the remote machine in real time. //! //! Uses the `enigo` crate for cross-platform input injection: //! - Windows: SendInput Win32 API //! - macOS: CoreGraphics CGEvent //! - Linux: X11 XTest extension use anyhow::Result; use enigo::{Button, Direction, Enigo, Key, Settings, Coordinate}; use log::{info, warn}; /// Manages input simulation on the local machine. pub struct InputHandler { enigo: Enigo, screen_width: u32, screen_height: u32, } impl InputHandler { /// Create a new input handler. /// /// `screen_width` and `screen_height` are the dimensions of the captured /// display. Mouse coordinates from viewers are in this coordinate space. pub fn new(screen_width: u32, screen_height: u32) -> Result { let enigo = Enigo::new(&Settings::default()) .map_err(|e| anyhow::anyhow!("failed to initialize input handler: {}", e))?; info!( "input handler initialized (screen {}x{})", screen_width, screen_height ); Ok(Self { enigo, screen_width, screen_height, }) } /// Execute a HUD command received from a viewer. /// /// Commands: /// - `mouse_move` — Move the cursor to (x, y). /// - `mouse_down` — Press a mouse button (left/middle/right). /// - `mouse_up` — Release a mouse button. /// - `mouse_click` — Click a mouse button (press + release). /// - `mouse_dblclick` — Double-click a mouse button. /// - `scroll` — Scroll vertically/horizontally. /// - `key_down` — Press a key. /// - `key_up` — Release a key. /// - `key_click` — Type a key (press + release). /// - `key_type` — Type a string of characters. pub fn execute(&mut self, command: &str, params: &serde_json::Value) -> Result<()> { match command { "mouse_move" => self.mouse_move(params), "mouse_down" => self.mouse_down(params), "mouse_up" => self.mouse_up(params), "mouse_click" => self.mouse_click(params), "mouse_dblclick" => self.mouse_double_click(params), "scroll" => self.scroll(params), "key_down" => self.key_down(params), "key_up" => self.key_up(params), "key_click" => self.key_click(params), "key_type" => self.key_type(params), _ => { warn!("unknown HUD command: {}", command); Ok(()) } } } // ── Mouse commands ───────────────────────────────────────────────────── /// Move the mouse cursor to the specified position. /// /// Params: `{ "x": number, "y": number }` fn mouse_move(&mut self, params: &serde_json::Value) -> Result<()> { let x = params["x"].as_i64().unwrap_or(0) as i32; let y = params["y"].as_i64().unwrap_or(0) as i32; // Clamp to screen bounds. let x = x.clamp(0, self.screen_width as i32 - 1); let y = y.clamp(0, self.screen_height as i32 - 1); self.enigo .move_mouse(x, y, Coordinate::Abs) .map_err(|e| anyhow::anyhow!("mouse_move failed: {}", e))?; Ok(()) } /// Press a mouse button down. /// /// Params: `{ "button": number }` — 0=left, 1=middle, 2=right fn mouse_down(&mut self, params: &serde_json::Value) -> Result<()> { let button = json_to_button(params["button"].as_i64().unwrap_or(0)); self.enigo .button(button, Direction::Press) .map_err(|e| anyhow::anyhow!("mouse_down failed: {}", e))?; Ok(()) } /// Release a mouse button. fn mouse_up(&mut self, params: &serde_json::Value) -> Result<()> { let button = json_to_button(params["button"].as_i64().unwrap_or(0)); self.enigo .button(button, Direction::Release) .map_err(|e| anyhow::anyhow!("mouse_up failed: {}", e))?; Ok(()) } /// Click a mouse button (press + release). fn mouse_click(&mut self, params: &serde_json::Value) -> Result<()> { let button = json_to_button(params["button"].as_i64().unwrap_or(0)); self.enigo .button(button, Direction::Click) .map_err(|e| anyhow::anyhow!("mouse_click failed: {}", e))?; Ok(()) } /// Double-click a mouse button. fn mouse_double_click(&mut self, params: &serde_json::Value) -> Result<()> { let button = json_to_button(params["button"].as_i64().unwrap_or(0)); // Double-click = two rapid clicks. self.enigo .button(button, Direction::Click) .map_err(|e| anyhow::anyhow!("mouse_dblclick (1st) failed: {}", e))?; std::thread::sleep(std::time::Duration::from_millis(30)); self.enigo .button(button, Direction::Click) .map_err(|e| anyhow::anyhow!("mouse_dblclick (2nd) failed: {}", e))?; Ok(()) } /// Scroll the mouse wheel. /// /// Params: `{ "deltaX": number, "deltaY": number }` /// Positive deltaY = scroll down, negative = scroll up. fn scroll(&mut self, params: &serde_json::Value) -> Result<()> { let delta_y = params["deltaY"].as_i64().unwrap_or(0) as i32; let delta_x = params["deltaX"].as_i64().unwrap_or(0) as i32; // Vertical scroll. if delta_y != 0 { let direction = if delta_y > 0 { Direction::ScrollDown } else { Direction::ScrollUp }; let amount = delta_y.abs().min(50); // Clamp to prevent crazy scrolling. for _ in 0..amount { self.enigo .scroll_y(1, direction) .map_err(|e| anyhow::anyhow!("scroll_y failed: {}", e))?; } } // Horizontal scroll. if delta_x != 0 { let direction = if delta_x > 0 { Direction::ScrollRight } else { Direction::ScrollLeft }; let amount = delta_x.abs().min(50); for _ in 0..amount { self.enigo .scroll_x(1, direction) .map_err(|e| anyhow::anyhow!("scroll_x failed: {}", e))?; } } Ok(()) } // ── Keyboard commands ────────────────────────────────────────────────── /// Press a key down. /// /// Params: `{ "key": string, "code": string, "ctrl": bool, "shift": bool, "alt": bool, "meta": bool }` fn key_down(&mut self, params: &serde_json::Value) -> Result<()> { let key = map_key( params["code"].as_str().unwrap_or(""), params["key"].as_str().unwrap_or(""), ); // Hold modifiers first (order matters for some combinations). if params["ctrl"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Control, Direction::Press); } if params["shift"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Shift, Direction::Press); } if params["alt"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Alt, Direction::Press); } if params["meta"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Meta, Direction::Press); } self.enigo .key(key, Direction::Press) .map_err(|e| anyhow::anyhow!("key_down failed: {}", e))?; Ok(()) } /// Release a key. fn key_up(&mut self, params: &serde_json::Value) -> Result<()> { let key = map_key( params["code"].as_str().unwrap_or(""), params["key"].as_str().unwrap_or(""), ); // Release the main key first. let _ = self.enigo.key(key, Direction::Release); // Release modifiers in reverse order. if params["meta"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Meta, Direction::Release); } if params["alt"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Alt, Direction::Release); } if params["shift"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Shift, Direction::Release); } if params["ctrl"].as_bool().unwrap_or(false) { let _ = self.enigo.key(Key::Control, Direction::Release); } Ok(()) } /// Type a single key (press + release). fn key_click(&mut self, params: &serde_json::Value) -> Result<()> { let key = map_key( params["code"].as_str().unwrap_or(""), params["key"].as_str().unwrap_or(""), ); self.enigo .key(key, Direction::Click) .map_err(|e| anyhow::anyhow!("key_click failed: {}", e))?; Ok(()) } /// Type a string of characters. /// /// Params: `{ "text": string }` fn key_type(&mut self, params: &serde_json::Value) -> Result<()> { let text = params["text"].as_str().unwrap_or(""); for ch in text.chars() { // Skip control characters. if ch.is_control() { continue; } if let Err(e) = self.enigo.key(Key::Unicode(ch), Direction::Click) { warn!("key_type failed for '{}': {}", ch, e); } } Ok(()) } } // ── Helper functions ────────────────────────────────────────────────────────── /// Convert a numeric button index to an `enigo::Button`. /// /// 0 = Left, 1 = Middle, 2 = Right, 3 = Back, 4 = Forward fn json_to_button(idx: i64) -> Button { match idx { 1 => Button::Middle, 2 => Button::Right, 3 => Button::Back, 4 => Button::Forward, _ => Button::Left, } } /// Map a browser keyboard event to an `enigo::Key`. /// /// `code` is the physical key position (e.g., "KeyA", "ShiftLeft", "Digit1"). /// `key` is the logical key value (e.g., "a", "Shift", "1"). /// /// Strategy: /// 1. If `key` is a single printable character, use `Key::Unicode(char)`. /// 2. Otherwise, map `code` to the corresponding `Key` variant. fn map_key(code: &str, key: &str) -> Key { // Printable single character — use Unicode for international keyboard support. if key.chars().count() == 1 { let ch = key.chars().next().unwrap(); if !ch.is_control() { return Key::Unicode(ch); } } // Map physical key codes to enigo Key variants. match code { // ── Modifier keys ────────────────────────────────────────────── "ShiftLeft" | "ShiftRight" => Key::Shift, "ControlLeft" | "ControlRight" => Key::Control, "AltLeft" | "AltRight" => Key::Alt, "MetaLeft" | "MetaRight" => Key::Meta, // ── Whitespace / editing ──────────────────────────────────────── "Backspace" => Key::Backspace, "Tab" => Key::Tab, "Enter" => Key::Return, "Space" => Key::Space, "Escape" => Key::Escape, // ── Arrow keys ────────────────────────────────────────────────── "ArrowUp" => Key::Up, "ArrowDown" => Key::Down, "ArrowLeft" => Key::Left, "ArrowRight" => Key::Right, // ── Function keys ─────────────────────────────────────────────── "F1" => Key::F1, "F2" => Key::F2, "F3" => Key::F3, "F4" => Key::F4, "F5" => Key::F5, "F6" => Key::F6, "F7" => Key::F7, "F8" => Key::F8, "F9" => Key::F9, "F10" => Key::F10, "F11" => Key::F11, "F12" => Key::F12, "F13" => Key::F13, "F14" => Key::F14, "F15" => Key::F15, "F16" => Key::F16, "F17" => Key::F17, "F18" => Key::F18, "F19" => Key::F19, "F20" => Key::F20, // ── Lock keys ─────────────────────────────────────────────────── "CapsLock" => Key::CapsLock, "NumLock" => Key::NumLock, "ScrollLock" => Key::ScrollLock, // ── Navigation / editing cluster ──────────────────────────────── "Insert" => Key::Insert, "Delete" => Key::Delete, "Home" => Key::Home, "End" => Key::End, "PageUp" => Key::PageUp, "PageDown" => Key::PageDown, "PrintScreen" => Key::PrintScreen, "Pause" => Key::Pause, // ── Numpad ────────────────────────────────────────────────────── "Numpad0" => Key::Numpad0, "Numpad1" => Key::Numpad1, "Numpad2" => Key::Numpad2, "Numpad3" => Key::Numpad3, "Numpad4" => Key::Numpad4, "Numpad5" => Key::Numpad5, "Numpad6" => Key::Numpad6, "Numpad7" => Key::Numpad7, "Numpad8" => Key::Numpad8, "Numpad9" => Key::Numpad9, "NumpadAdd" => Key::NumpadAdd, "NumpadSubtract" => Key::NumpadSubtract, "NumpadMultiply" => Key::NumpadMultiply, "NumpadDivide" => Key::NumpadDivide, "NumpadDecimal" => Key::NumpadDecimal, "NumpadEnter" => Key::NumpadEnter, "NumLock" => Key::NumLock, // ── Punctuation / symbols ─────────────────────────────────────── // These are handled as Unicode characters when sent via `key` field. // If only `code` is available, fall back to Unicode mapping. "BracketLeft" => Key::Unicode('['), "BracketRight" => Key::Unicode(']'), "Semicolon" => Key::Unicode(';'), "Quote" => Key::Unicode('\''), "Backquote" => Key::Unicode('`'), "Backslash" => Key::Unicode('\\'), "Slash" => Key::Unicode('/'), "Comma" => Key::Unicode(','), "Period" => Key::Unicode('.'), "Minus" => Key::Unicode('-'), "Equal" => Key::Unicode('='), // ── Fallback: try to use the key value as Unicode ─────────────── _ => { if !key.is_empty() && key.chars().count() == 1 { Key::Unicode(key.chars().next().unwrap()) } else { warn!("unmapped key code '{}', key '{}'", code, key); Key::Unicode('\0') // No-op key } } } }