agent: input.rs — full remote control: mouse move/click/dblclick/scroll, keyboard with 60+ key mappings
This commit is contained in:
Butterfly Dev
parent
4c93b47a5d
commit
e1e6442ff5
406
agent/src/input.rs
Normal file
406
agent/src/input.rs
Normal file
@ -0,0 +1,406 @@
|
|||||||
|
//! Input simulation module.
|
||||||
|
//!
|
||||||
|
//! Receives HUD commands from the Butterfly server (forwarded from browser viewers)
|
||||||
|
//! and executes them as real mouse/keyboard events on the local machine. This is
|
||||||
|
//! the core of the remote control functionality — a viewer in the browser can
|
||||||
|
//! move the mouse, click, type, and scroll on the remote machine in real time.
|
||||||
|
//!
|
||||||
|
//! Uses the `enigo` crate for cross-platform input injection:
|
||||||
|
//! - Windows: SendInput Win32 API
|
||||||
|
//! - macOS: CoreGraphics CGEvent
|
||||||
|
//! - Linux: X11 XTest extension
|
||||||
|
|
||||||
|
use anyhow::Result;
|
||||||
|
use enigo::{Button, Direction, Enigo, Key, Settings, Coordinate};
|
||||||
|
use log::{info, warn};
|
||||||
|
|
||||||
|
/// Manages input simulation on the local machine.
|
||||||
|
pub struct InputHandler {
|
||||||
|
enigo: Enigo,
|
||||||
|
screen_width: u32,
|
||||||
|
screen_height: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InputHandler {
|
||||||
|
/// Create a new input handler.
|
||||||
|
///
|
||||||
|
/// `screen_width` and `screen_height` are the dimensions of the captured
|
||||||
|
/// display. Mouse coordinates from viewers are in this coordinate space.
|
||||||
|
pub fn new(screen_width: u32, screen_height: u32) -> Result<Self> {
|
||||||
|
let enigo = Enigo::new(&Settings::default())
|
||||||
|
.map_err(|e| anyhow::anyhow!("failed to initialize input handler: {}", e))?;
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"input handler initialized (screen {}x{})",
|
||||||
|
screen_width, screen_height
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
enigo,
|
||||||
|
screen_width,
|
||||||
|
screen_height,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Execute a HUD command received from a viewer.
|
||||||
|
///
|
||||||
|
/// Commands:
|
||||||
|
/// - `mouse_move` — Move the cursor to (x, y).
|
||||||
|
/// - `mouse_down` — Press a mouse button (left/middle/right).
|
||||||
|
/// - `mouse_up` — Release a mouse button.
|
||||||
|
/// - `mouse_click` — Click a mouse button (press + release).
|
||||||
|
/// - `mouse_dblclick` — Double-click a mouse button.
|
||||||
|
/// - `scroll` — Scroll vertically/horizontally.
|
||||||
|
/// - `key_down` — Press a key.
|
||||||
|
/// - `key_up` — Release a key.
|
||||||
|
/// - `key_click` — Type a key (press + release).
|
||||||
|
/// - `key_type` — Type a string of characters.
|
||||||
|
pub fn execute(&mut self, command: &str, params: &serde_json::Value) -> Result<()> {
|
||||||
|
match command {
|
||||||
|
"mouse_move" => self.mouse_move(params),
|
||||||
|
"mouse_down" => self.mouse_down(params),
|
||||||
|
"mouse_up" => self.mouse_up(params),
|
||||||
|
"mouse_click" => self.mouse_click(params),
|
||||||
|
"mouse_dblclick" => self.mouse_double_click(params),
|
||||||
|
"scroll" => self.scroll(params),
|
||||||
|
"key_down" => self.key_down(params),
|
||||||
|
"key_up" => self.key_up(params),
|
||||||
|
"key_click" => self.key_click(params),
|
||||||
|
"key_type" => self.key_type(params),
|
||||||
|
_ => {
|
||||||
|
warn!("unknown HUD command: {}", command);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Mouse commands ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Move the mouse cursor to the specified position.
|
||||||
|
///
|
||||||
|
/// Params: `{ "x": number, "y": number }`
|
||||||
|
fn mouse_move(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let x = params["x"].as_i64().unwrap_or(0) as i32;
|
||||||
|
let y = params["y"].as_i64().unwrap_or(0) as i32;
|
||||||
|
|
||||||
|
// Clamp to screen bounds.
|
||||||
|
let x = x.clamp(0, self.screen_width as i32 - 1);
|
||||||
|
let y = y.clamp(0, self.screen_height as i32 - 1);
|
||||||
|
|
||||||
|
self.enigo
|
||||||
|
.move_mouse(x, y, Coordinate::Abs)
|
||||||
|
.map_err(|e| anyhow::anyhow!("mouse_move failed: {}", e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Press a mouse button down.
|
||||||
|
///
|
||||||
|
/// Params: `{ "button": number }` — 0=left, 1=middle, 2=right
|
||||||
|
fn mouse_down(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let button = json_to_button(params["button"].as_i64().unwrap_or(0));
|
||||||
|
self.enigo
|
||||||
|
.button(button, Direction::Press)
|
||||||
|
.map_err(|e| anyhow::anyhow!("mouse_down failed: {}", e))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Release a mouse button.
|
||||||
|
fn mouse_up(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let button = json_to_button(params["button"].as_i64().unwrap_or(0));
|
||||||
|
self.enigo
|
||||||
|
.button(button, Direction::Release)
|
||||||
|
.map_err(|e| anyhow::anyhow!("mouse_up failed: {}", e))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Click a mouse button (press + release).
|
||||||
|
fn mouse_click(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let button = json_to_button(params["button"].as_i64().unwrap_or(0));
|
||||||
|
self.enigo
|
||||||
|
.button(button, Direction::Click)
|
||||||
|
.map_err(|e| anyhow::anyhow!("mouse_click failed: {}", e))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Double-click a mouse button.
|
||||||
|
fn mouse_double_click(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let button = json_to_button(params["button"].as_i64().unwrap_or(0));
|
||||||
|
// Double-click = two rapid clicks.
|
||||||
|
self.enigo
|
||||||
|
.button(button, Direction::Click)
|
||||||
|
.map_err(|e| anyhow::anyhow!("mouse_dblclick (1st) failed: {}", e))?;
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||||
|
self.enigo
|
||||||
|
.button(button, Direction::Click)
|
||||||
|
.map_err(|e| anyhow::anyhow!("mouse_dblclick (2nd) failed: {}", e))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scroll the mouse wheel.
|
||||||
|
///
|
||||||
|
/// Params: `{ "deltaX": number, "deltaY": number }`
|
||||||
|
/// Positive deltaY = scroll down, negative = scroll up.
|
||||||
|
fn scroll(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let delta_y = params["deltaY"].as_i64().unwrap_or(0) as i32;
|
||||||
|
let delta_x = params["deltaX"].as_i64().unwrap_or(0) as i32;
|
||||||
|
|
||||||
|
// Vertical scroll.
|
||||||
|
if delta_y != 0 {
|
||||||
|
let direction = if delta_y > 0 {
|
||||||
|
Direction::ScrollDown
|
||||||
|
} else {
|
||||||
|
Direction::ScrollUp
|
||||||
|
};
|
||||||
|
let amount = delta_y.abs().min(50); // Clamp to prevent crazy scrolling.
|
||||||
|
for _ in 0..amount {
|
||||||
|
self.enigo
|
||||||
|
.scroll_y(1, direction)
|
||||||
|
.map_err(|e| anyhow::anyhow!("scroll_y failed: {}", e))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Horizontal scroll.
|
||||||
|
if delta_x != 0 {
|
||||||
|
let direction = if delta_x > 0 {
|
||||||
|
Direction::ScrollRight
|
||||||
|
} else {
|
||||||
|
Direction::ScrollLeft
|
||||||
|
};
|
||||||
|
let amount = delta_x.abs().min(50);
|
||||||
|
for _ in 0..amount {
|
||||||
|
self.enigo
|
||||||
|
.scroll_x(1, direction)
|
||||||
|
.map_err(|e| anyhow::anyhow!("scroll_x failed: {}", e))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Keyboard commands ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Press a key down.
|
||||||
|
///
|
||||||
|
/// Params: `{ "key": string, "code": string, "ctrl": bool, "shift": bool, "alt": bool, "meta": bool }`
|
||||||
|
fn key_down(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let key = map_key(
|
||||||
|
params["code"].as_str().unwrap_or(""),
|
||||||
|
params["key"].as_str().unwrap_or(""),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Hold modifiers first (order matters for some combinations).
|
||||||
|
if params["ctrl"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Control, Direction::Press);
|
||||||
|
}
|
||||||
|
if params["shift"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Shift, Direction::Press);
|
||||||
|
}
|
||||||
|
if params["alt"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Alt, Direction::Press);
|
||||||
|
}
|
||||||
|
if params["meta"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Meta, Direction::Press);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.enigo
|
||||||
|
.key(key, Direction::Press)
|
||||||
|
.map_err(|e| anyhow::anyhow!("key_down failed: {}", e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Release a key.
|
||||||
|
fn key_up(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let key = map_key(
|
||||||
|
params["code"].as_str().unwrap_or(""),
|
||||||
|
params["key"].as_str().unwrap_or(""),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Release the main key first.
|
||||||
|
let _ = self.enigo.key(key, Direction::Release);
|
||||||
|
|
||||||
|
// Release modifiers in reverse order.
|
||||||
|
if params["meta"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Meta, Direction::Release);
|
||||||
|
}
|
||||||
|
if params["alt"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Alt, Direction::Release);
|
||||||
|
}
|
||||||
|
if params["shift"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Shift, Direction::Release);
|
||||||
|
}
|
||||||
|
if params["ctrl"].as_bool().unwrap_or(false) {
|
||||||
|
let _ = self.enigo.key(Key::Control, Direction::Release);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Type a single key (press + release).
|
||||||
|
fn key_click(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let key = map_key(
|
||||||
|
params["code"].as_str().unwrap_or(""),
|
||||||
|
params["key"].as_str().unwrap_or(""),
|
||||||
|
);
|
||||||
|
|
||||||
|
self.enigo
|
||||||
|
.key(key, Direction::Click)
|
||||||
|
.map_err(|e| anyhow::anyhow!("key_click failed: {}", e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Type a string of characters.
|
||||||
|
///
|
||||||
|
/// Params: `{ "text": string }`
|
||||||
|
fn key_type(&mut self, params: &serde_json::Value) -> Result<()> {
|
||||||
|
let text = params["text"].as_str().unwrap_or("");
|
||||||
|
for ch in text.chars() {
|
||||||
|
// Skip control characters.
|
||||||
|
if ch.is_control() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Err(e) = self.enigo.key(Key::Unicode(ch), Direction::Click) {
|
||||||
|
warn!("key_type failed for '{}': {}", ch, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helper functions ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Convert a numeric button index to an `enigo::Button`.
|
||||||
|
///
|
||||||
|
/// 0 = Left, 1 = Middle, 2 = Right, 3 = Back, 4 = Forward
|
||||||
|
fn json_to_button(idx: i64) -> Button {
|
||||||
|
match idx {
|
||||||
|
1 => Button::Middle,
|
||||||
|
2 => Button::Right,
|
||||||
|
3 => Button::Back,
|
||||||
|
4 => Button::Forward,
|
||||||
|
_ => Button::Left,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map a browser keyboard event to an `enigo::Key`.
|
||||||
|
///
|
||||||
|
/// `code` is the physical key position (e.g., "KeyA", "ShiftLeft", "Digit1").
|
||||||
|
/// `key` is the logical key value (e.g., "a", "Shift", "1").
|
||||||
|
///
|
||||||
|
/// Strategy:
|
||||||
|
/// 1. If `key` is a single printable character, use `Key::Unicode(char)`.
|
||||||
|
/// 2. Otherwise, map `code` to the corresponding `Key` variant.
|
||||||
|
fn map_key(code: &str, key: &str) -> Key {
|
||||||
|
// Printable single character — use Unicode for international keyboard support.
|
||||||
|
if key.chars().count() == 1 {
|
||||||
|
let ch = key.chars().next().unwrap();
|
||||||
|
if !ch.is_control() {
|
||||||
|
return Key::Unicode(ch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map physical key codes to enigo Key variants.
|
||||||
|
match code {
|
||||||
|
// ── Modifier keys ──────────────────────────────────────────────
|
||||||
|
"ShiftLeft" | "ShiftRight" => Key::Shift,
|
||||||
|
"ControlLeft" | "ControlRight" => Key::Control,
|
||||||
|
"AltLeft" | "AltRight" => Key::Alt,
|
||||||
|
"MetaLeft" | "MetaRight" => Key::Meta,
|
||||||
|
|
||||||
|
// ── Whitespace / editing ────────────────────────────────────────
|
||||||
|
"Backspace" => Key::Backspace,
|
||||||
|
"Tab" => Key::Tab,
|
||||||
|
"Enter" => Key::Return,
|
||||||
|
"Space" => Key::Space,
|
||||||
|
"Escape" => Key::Escape,
|
||||||
|
|
||||||
|
// ── Arrow keys ──────────────────────────────────────────────────
|
||||||
|
"ArrowUp" => Key::Up,
|
||||||
|
"ArrowDown" => Key::Down,
|
||||||
|
"ArrowLeft" => Key::Left,
|
||||||
|
"ArrowRight" => Key::Right,
|
||||||
|
|
||||||
|
// ── Function keys ───────────────────────────────────────────────
|
||||||
|
"F1" => Key::F1,
|
||||||
|
"F2" => Key::F2,
|
||||||
|
"F3" => Key::F3,
|
||||||
|
"F4" => Key::F4,
|
||||||
|
"F5" => Key::F5,
|
||||||
|
"F6" => Key::F6,
|
||||||
|
"F7" => Key::F7,
|
||||||
|
"F8" => Key::F8,
|
||||||
|
"F9" => Key::F9,
|
||||||
|
"F10" => Key::F10,
|
||||||
|
"F11" => Key::F11,
|
||||||
|
"F12" => Key::F12,
|
||||||
|
"F13" => Key::F13,
|
||||||
|
"F14" => Key::F14,
|
||||||
|
"F15" => Key::F15,
|
||||||
|
"F16" => Key::F16,
|
||||||
|
"F17" => Key::F17,
|
||||||
|
"F18" => Key::F18,
|
||||||
|
"F19" => Key::F19,
|
||||||
|
"F20" => Key::F20,
|
||||||
|
|
||||||
|
// ── Lock keys ───────────────────────────────────────────────────
|
||||||
|
"CapsLock" => Key::CapsLock,
|
||||||
|
"NumLock" => Key::NumLock,
|
||||||
|
"ScrollLock" => Key::ScrollLock,
|
||||||
|
|
||||||
|
// ── Navigation / editing cluster ────────────────────────────────
|
||||||
|
"Insert" => Key::Insert,
|
||||||
|
"Delete" => Key::Delete,
|
||||||
|
"Home" => Key::Home,
|
||||||
|
"End" => Key::End,
|
||||||
|
"PageUp" => Key::PageUp,
|
||||||
|
"PageDown" => Key::PageDown,
|
||||||
|
"PrintScreen" => Key::PrintScreen,
|
||||||
|
"Pause" => Key::Pause,
|
||||||
|
|
||||||
|
// ── Numpad ──────────────────────────────────────────────────────
|
||||||
|
"Numpad0" => Key::Numpad0,
|
||||||
|
"Numpad1" => Key::Numpad1,
|
||||||
|
"Numpad2" => Key::Numpad2,
|
||||||
|
"Numpad3" => Key::Numpad3,
|
||||||
|
"Numpad4" => Key::Numpad4,
|
||||||
|
"Numpad5" => Key::Numpad5,
|
||||||
|
"Numpad6" => Key::Numpad6,
|
||||||
|
"Numpad7" => Key::Numpad7,
|
||||||
|
"Numpad8" => Key::Numpad8,
|
||||||
|
"Numpad9" => Key::Numpad9,
|
||||||
|
"NumpadAdd" => Key::NumpadAdd,
|
||||||
|
"NumpadSubtract" => Key::NumpadSubtract,
|
||||||
|
"NumpadMultiply" => Key::NumpadMultiply,
|
||||||
|
"NumpadDivide" => Key::NumpadDivide,
|
||||||
|
"NumpadDecimal" => Key::NumpadDecimal,
|
||||||
|
"NumpadEnter" => Key::NumpadEnter,
|
||||||
|
"NumLock" => Key::NumLock,
|
||||||
|
|
||||||
|
// ── Punctuation / symbols ───────────────────────────────────────
|
||||||
|
// These are handled as Unicode characters when sent via `key` field.
|
||||||
|
// If only `code` is available, fall back to Unicode mapping.
|
||||||
|
"BracketLeft" => Key::Unicode('['),
|
||||||
|
"BracketRight" => Key::Unicode(']'),
|
||||||
|
"Semicolon" => Key::Unicode(';'),
|
||||||
|
"Quote" => Key::Unicode('\''),
|
||||||
|
"Backquote" => Key::Unicode('`'),
|
||||||
|
"Backslash" => Key::Unicode('\\'),
|
||||||
|
"Slash" => Key::Unicode('/'),
|
||||||
|
"Comma" => Key::Unicode(','),
|
||||||
|
"Period" => Key::Unicode('.'),
|
||||||
|
"Minus" => Key::Unicode('-'),
|
||||||
|
"Equal" => Key::Unicode('='),
|
||||||
|
|
||||||
|
// ── Fallback: try to use the key value as Unicode ───────────────
|
||||||
|
_ => {
|
||||||
|
if !key.is_empty() && key.chars().count() == 1 {
|
||||||
|
Key::Unicode(key.chars().next().unwrap())
|
||||||
|
} else {
|
||||||
|
warn!("unmapped key code '{}', key '{}'", code, key);
|
||||||
|
Key::Unicode('\0') // No-op key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user