roboattic Lab
Published © LGPL

Gesture Control Mouse

A Node MCU, MPU 6050, and flex sensor create a gesture-controlled mouse for smooth PC control, ideal for school and college projects.

IntermediateFull instructions provided5 hours46
Gesture Control Mouse

Things used in this project

Software apps and online services

Arduino IDE
Arduino IDE
VS Code
Microsoft VS Code

Story

Read more

Schematics

Circuit Diagram

Code

Final Node MCU Code

C/C++
#include <ESP8266WiFi.h>
#include <ESP8266WebServer.h>
#include <Wire.h>
#include <Adafruit_MPU6050.h>
#include <Adafruit_Sensor.h>

// WiFi credentials
const char* ssid = "Wifi Name";  
const char* password = "Password";
+

// Flex sensor setup
const int flexPin = A0;                // NodeMCU analog pin A0
const float VCC = 3.3;                 // NodeMCU voltage (3.3V)
const float R_DIV = 47000.0;           // 47K ohm resistor
const float flatResistance = 25000.0;  // Resistance when flat
const float bendResistance = 100000.0; // Resistance at 90 deg

// Click detection variables
const float CLICK_MIN_ANGLE = 60.0;    // Minimum angle to trigger a click
const float CLICK_MAX_ANGLE = 90.0;    // Maximum angle to consider
const unsigned long CLICK_COOLDOWN = 700; // Milliseconds to wait before registering another click
boolean isClicked = false;             // Current click state
boolean clickSent = false;             // Whether a click was already sent for this bend
unsigned long lastClickTime = 0;       // When the last click occurred

ESP8266WebServer server(80);
Adafruit_MPU6050 mpu;

// Debug flag
bool debug = true;

void setup() {
  // Initialize serial communication
  Serial.begin(115200);
  delay(100);  // Short delay for serial to initialize
  
  Serial.println("\n\n=== Gesture Mouse Control System ===");
  Serial.println("Initializing...");
  
  // Initialize I2C for MPU6050
  Wire.begin(D2, D1);  // SDA=D2, SCL=D1
  delay(50);  // Give some time for I2C to initialize

  // Initialize MPU6050
  Serial.println("Connecting to MPU6050...");
  bool mpuInitialized = false;
  for (int i = 0; i < 5; i++) {  // Try 5 times to initialize
    if (mpu.begin()) {
      mpuInitialized = true;
      break;
    }
    Serial.println("Failed to find MPU6050 chip. Retrying...");
    delay(500);
  }
  
  if (!mpuInitialized) {
    Serial.println("ERROR: Could not connect to MPU6050 sensor!");
    Serial.println("Please check your wiring and restart the device.");
    while (1) {
      delay(10);
    }
  }
  
  Serial.println("MPU6050 Found and Initialized Successfully!");
  
  // Configure MPU6050 sensor settings
  mpu.setAccelerometerRange(MPU6050_RANGE_8_G);
  mpu.setGyroRange(MPU6050_RANGE_500_DEG);
  mpu.setFilterBandwidth(MPU6050_BAND_21_HZ);
  
  // Test reading from flex sensor
  Serial.println("Testing flex sensor...");
  int flexReading = analogRead(flexPin);
  Serial.print("Flex sensor raw reading: ");
  Serial.println(flexReading);
  if (flexReading == 0 || flexReading > 1020) {
    Serial.println("WARNING: Flex sensor reading seems invalid. Check your connections!");
  } else {
    Serial.println("Flex sensor reading looks good!");
  }
  
  // Connect to WiFi
  WiFi.mode(WIFI_STA);  // Set WiFi to station mode
  WiFi.disconnect();    // Disconnect from any previous connections
  delay(100);
  
  Serial.print("Connecting to WiFi network: ");
  Serial.println(ssid);
  
  WiFi.begin(ssid, password);
  
  // Wait for connection with timeout
  int timeout = 0;
  while (WiFi.status() != WL_CONNECTED && timeout < 20) {
    delay(500);
    Serial.print(".");
    timeout++;
  }
  
  if (WiFi.status() != WL_CONNECTED) {
    Serial.println("\nFAILED to connect to WiFi network!");
    Serial.println("Please check your WiFi credentials and signal strength.");
    Serial.println("The system will continue to function, but web server will not be available.");
  } else {
    Serial.println("");
    Serial.print("Connected successfully to WiFi network: ");
    Serial.println(ssid);
    Serial.print("IP address: ");
    Serial.println(WiFi.localIP());
    
    // Define server routes
    server.on("/", HTTP_GET, handleRoot);
    server.on("/data", HTTP_GET, handleData);
    server.onNotFound(handleNotFound);
    
    // Start server
    server.begin();
    Serial.println("HTTP server started on port 80");
    Serial.println("You can access the data endpoint at: http://" + WiFi.localIP().toString() + "/data");
  }
  
  Serial.println("System initialization complete. Starting main loop...");
}

void loop() {
  // Handle client requests if WiFi is connected
  if (WiFi.status() == WL_CONNECTED) {
    server.handleClient();
  } else {
    // If WiFi disconnected, try to reconnect occasionally
    static unsigned long lastReconnectAttempt = 0;
    unsigned long currentMillis = millis();
    
    if (currentMillis - lastReconnectAttempt > 30000) { // Try every 30 seconds
      lastReconnectAttempt = currentMillis;
      
      if (WiFi.status() != WL_CONNECTED) {
        Serial.println("WiFi disconnected. Attempting to reconnect...");
        WiFi.reconnect();
      }
    }
  }
  
  // Always update the flex sensor state regardless of WiFi status
  updateFlexSensorState();
  
  // Small delay to prevent CPU hogging
  delay(10);
}

// Custom map function for float values
float mapf(float x, float in_min, float in_max, float out_min, float out_max) {
  return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min;
}

void updateFlexSensorState() {
  // Read the flex sensor
  int ADCflex = analogRead(flexPin);
  
  // Calculate voltage and resistance
  float Vflex = ADCflex * VCC / 1023.0;
  float Rflex = R_DIV * (VCC / Vflex - 1.0);
  
  // Calculate the bend angle
  float angle = mapf(Rflex, flatResistance, bendResistance, 0, 90.0);
  
  // Constrain angle to avoid out-of-range values
  angle = constrain(angle, 0, 90.0);
  
  // Get current time
  unsigned long currentTime = millis();
  
  // Debug output (uncomment if needed)
  static unsigned long lastDebugOutput = 0;
  if (debug && (currentTime - lastDebugOutput > 1000)) {
    lastDebugOutput = currentTime;
    Serial.print("Flex Sensor - Raw: ");
    Serial.print(ADCflex);
    Serial.print(", Angle: ");
    Serial.print(angle);
    Serial.print(" degrees, Click state: ");
    Serial.println(isClicked ? "TRUE" : "false");
  }
  
  // Check if angle is in the click range
  if (angle >= CLICK_MIN_ANGLE && angle <= CLICK_MAX_ANGLE) {
    // If not already clicked and enough time has passed since last click
    if (!clickSent && (currentTime - lastClickTime > CLICK_COOLDOWN)) {
      isClicked = true;
      clickSent = true;
      lastClickTime = currentTime;
      Serial.println("CLICK EVENT DETECTED! Angle: " + String(angle));
    }
  } else {
    // Reset click sent flag when no longer in click range
    clickSent = false;
    isClicked = false;
  }
}

void handleRoot() {
  String html = "<!DOCTYPE html>\n";
  html += "<html>\n";
  html += "<head>\n";
  html += "<title>Gesture Mouse Control</title>\n";
  html += "<meta http-equiv='refresh' content='5'>\n"; // Auto-refresh page every 5 seconds
  html += "<style>\n";
  html += "body { font-family: Arial, sans-serif; margin: 20px; background-color: #f0f0f0; }\n";
  html += "h1 { color: #0066cc; }\n";
  html += ".status { padding: 10px; background-color: #e0e0e0; border-radius: 5px; margin-top: 20px; }\n";
  html += ".data { font-family: monospace; margin-top: 20px; }\n";
  html += "</style>\n";
  html += "</head>\n";
  html += "<body>\n";
  html += "<h1>Gesture Mouse Control System</h1>\n";
  
  html += "<div class='status'>\n";
  html += "<p>System Status: <strong>Running</strong></p>\n";
  html += "<p>Server IP: <strong>" + WiFi.localIP().toString() + "</strong></p>\n";
  html += "<p>Get sensor data at: <a href='/data'>/data</a> (JSON format)</p>\n";
  html += "</div>\n";

  // Get current sensor data
  sensors_event_t a, g, temp;
  mpu.getEvent(&a, &g, &temp);
  
  int ADCflex = analogRead(flexPin);
  float Vflex = ADCflex * VCC / 1023.0;
  float Rflex = R_DIV * (VCC / Vflex - 1.0);
  float angle = mapf(Rflex, flatResistance, bendResistance, 0, 90.0);
  angle = constrain(angle, 0, 90.0);
  
  html += "<div class='data'>\n";
  html += "<h2>Current Sensor Data:</h2>\n";
  
  html += "<p>Accelerometer (m/s²):<br>\n";
  html += "X: " + String(a.acceleration.x, 2) + "<br>\n";
  html += "Y: " + String(a.acceleration.y, 2) + "<br>\n";
  html += "Z: " + String(a.acceleration.z, 2) + "</p>\n";
  
  html += "<p>Gyroscope (rad/s):<br>\n";
  html += "X: " + String(g.gyro.x, 2) + "<br>\n";
  html += "Y: " + String(g.gyro.y, 2) + "<br>\n";
  html += "Z: " + String(g.gyro.z, 2) + "</p>\n";
  
  html += "<p>Flex Sensor:<br>\n";
  html += "Raw: " + String(ADCflex) + "<br>\n";
  html += "Angle: " + String(angle, 2) + " degrees<br>\n";
  html += "Click state: " + String(isClicked ? "TRUE" : "false") + "</p>\n";
  
  html += "<p>Temperature: " + String(temp.temperature, 2) + " °C</p>\n";
  html += "</div>\n";
  
  html += "</body>\n";
  html += "</html>\n";
  
  server.send(200, "text/html", html);
}

void handleData() {
  sensors_event_t a, g, temp;
  mpu.getEvent(&a, &g, &temp);
  
  // Read the flex sensor for the JSON data
  int ADCflex = analogRead(flexPin);
  float Vflex = ADCflex * VCC / 1023.0;
  float Rflex = R_DIV * (VCC / Vflex - 1.0);
  float angle = mapf(Rflex, flatResistance, bendResistance, 0, 90.0);
  angle = constrain(angle, 0, 90.0);
  
  // Enable CORS headers
  server.sendHeader("Access-Control-Allow-Origin", "*");
  server.sendHeader("Access-Control-Allow-Methods", "GET");
  server.sendHeader("Access-Control-Allow-Headers", "Content-Type");
  
  // Create JSON response
  String json = "{";
  json += "\"accelerometer\": {";
  json += "\"x\": " + String(a.acceleration.x) + ", ";
  json += "\"y\": " + String(a.acceleration.y) + ", ";
  json += "\"z\": " + String(a.acceleration.z);
  json += "}, ";
  json += "\"gyroscope\": {";
  json += "\"x\": " + String(g.gyro.x) + ", ";
  json += "\"y\": " + String(g.gyro.y) + ", ";
  json += "\"z\": " + String(g.gyro.z);
  json += "}, ";
  json += "\"flex\": {";
  json += "\"angle\": " + String(angle) + ", ";
  json += "\"raw\": " + String(ADCflex) + ", ";
  json += "\"click\": " + String(isClicked ? "true" : "false");
  json += "}, ";
  json += "\"temperature\": " + String(temp.temperature);
  json += "}";
  
  server.send(200, "application/json", json);
  
  // Debug output
  if (debug) {
    Serial.println("Data endpoint accessed - Sent JSON response");
  }
}

void handleNotFound() {
  String message = "File Not Found\n\n";
  message += "URI: ";
  message += server.uri();
  message += "\nMethod: ";
  message += (server.method() == HTTP_GET) ? "GET" : "POST";
  message += "\nArguments: ";
  message += server.args();
  message += "\n";
  
  for (uint8_t i = 0; i < server.args(); i++) {
    message += " " + server.argName(i) + ": " + server.arg(i) + "\n";
  }
  
  server.send(404, "text/plain", message);
}

Final Python Code

Python
import requests
import json
import time
import numpy as np
import pyautogui
import math
from collections import deque
import tkinter as tk
from tkinter import ttk
import threading
import sys
from tkinter import messagebox

NODEMCU_IP = '192.168.31.212'  # Replace with your NodeMCU IP
DATA_URL = f'http://{NODEMCU_IP}/data'
CONNECTION_TIMEOUT = 3
MAX_RETRIES = 5 

MOUSE_SENSITIVITY_X = 10.0  
MOUSE_SENSITIVITY_Y = 10.0
GYRO_THRESHOLD = 0.01  # Minimum gyro movement to register (eliminates jitter)
ACCELERATION_FACTOR = 2  # Exponential acceleration for faster movements
SCREEN_WIDTH, SCREEN_HEIGHT = pyautogui.size()
SMOOTHING_WINDOW = 5  # Number of samples for smoothing

# Movement modes
MODE_ABSOLUTE = "Absolute"  # Direct mapping of tilt to screen position
MODE_RELATIVE = "Relative"   # Tilt controls velocity, not position
MODE_HYBRID = "Hybrid"       # A mix of both with auto-centering

# For safety (prevent mouse from going crazy)
pyautogui.FAILSAFE = True  # Move mouse to corner to abort

# For UI
UPDATE_INTERVAL = 25  # milliseconds (faster updates)

# Global variables
is_paused = False
is_running = True
show_debug = True
baseline_accel = {"x": 0, "y": 0, "z": 0}
baseline_gyro = {"x": 0, "y": 0, "z": 0}
calibration_samples = 20
mouse_pos = {"x": pyautogui.position()[0], "y": pyautogui.position()[1]}
movement_mode = MODE_RELATIVE  # Start with relative mode by default

# Flex sensor readings
flex_angle = 0
flex_raw = 0

# Debug variables
last_data_time = 0
fps_counter = 0
fps = 0

# Smoothing buffers
accel_history = {"x": deque(maxlen=SMOOTHING_WINDOW), 
                "y": deque(maxlen=SMOOTHING_WINDOW), 
                "z": deque(maxlen=SMOOTHING_WINDOW)}
gyro_history = {"x": deque(maxlen=SMOOTHING_WINDOW), 
               "y": deque(maxlen=SMOOTHING_WINDOW), 
               "z": deque(maxlen=SMOOTHING_WINDOW)}

# Create UI
root = tk.Tk()
root.title("Gesture Mouse Control")
root.geometry("700x580")
root.protocol("WM_DELETE_WINDOW", lambda: set_running(False))

# Style configuration for a more modern look
style = ttk.Style()
style.configure("TButton", padding=6, relief="flat", background="#ccc")
style.configure("TLabel", padding=2)
style.configure("TFrame", padding=5)

def set_running(state):
    global is_running
    is_running = state
    if not state:
        root.destroy()
        sys.exit(0)

def toggle_pause():
    global is_paused
    is_paused = not is_paused
    pause_btn.config(text="Resume" if is_paused else "Pause")
    status_var.set("PAUSED" if is_paused else "RUNNING")

def toggle_debug():
    global show_debug
    show_debug = not show_debug
    debug_btn.config(text="Hide Debug" if show_debug else "Show Debug")
    if show_debug:
        debug_frame.pack(fill=tk.X, expand=True, pady=5)
    else:
        debug_frame.pack_forget()

def change_movement_mode(mode):
    global movement_mode
    movement_mode = mode
    status_var.set(f"Mode: {movement_mode}")
    
    # Update radio buttons
    mode_var.set(mode)
    
    # Clear movement history when changing modes
    for axis in ["x", "y", "z"]:
        accel_history[axis].clear()
        gyro_history[axis].clear()
    
def start_calibration():
    global baseline_accel, baseline_gyro
    status_var.set("Calibrating... Don't move the sensor!")
    root.update()
    
    # Reset baselines
    baseline_accel = {"x": 0, "y": 0, "z": 0}
    baseline_gyro = {"x": 0, "y": 0, "z": 0}
    
    # Collect samples
    accel_samples = {"x": [], "y": [], "z": []}
    gyro_samples = {"x": [], "y": [], "z": []}
    
    # Progress bar for calibration
    progress = ttk.Progressbar(status_frame, length=200, mode="determinate")
    progress.pack(pady=5)
    
    for i in range(calibration_samples):
        data = fetch_data()
        if data:
            for axis in ["x", "y", "z"]:
                accel_samples[axis].append(data["accelerometer"][axis])
                gyro_samples[axis].append(data["gyroscope"][axis])
            
            # Update progress bar
            progress["value"] = (i + 1) / calibration_samples * 100
            root.update_idletasks()
            time.sleep(0.1)
    
    # Calculate averages
    for axis in ["x", "y", "z"]:
        if accel_samples[axis]:  # Check if we got samples
            baseline_accel[axis] = sum(accel_samples[axis]) / len(accel_samples[axis])
            baseline_gyro[axis] = sum(gyro_samples[axis]) / len(gyro_samples[axis])
    
    progress.destroy()
    status_var.set("Calibration complete")
    
    # Update settings display
    sensitivity_x_var.set(MOUSE_SENSITIVITY_X)
    sensitivity_y_var.set(MOUSE_SENSITIVITY_Y)
    threshold_var.set(GYRO_THRESHOLD)
    acceleration_var.set(ACCELERATION_FACTOR)
    
    # Reset position to current mouse position
    global mouse_pos
    current_x, current_y = pyautogui.position()
    mouse_pos = {"x": current_x, "y": current_y}

def update_sensitivity():
    global MOUSE_SENSITIVITY_X, MOUSE_SENSITIVITY_Y, GYRO_THRESHOLD, ACCELERATION_FACTOR
    try:
        MOUSE_SENSITIVITY_X = float(sensitivity_x_var.get())
        MOUSE_SENSITIVITY_Y = float(sensitivity_y_var.get())
        GYRO_THRESHOLD = float(threshold_var.get())
        ACCELERATION_FACTOR = float(acceleration_var.get())
        status_var.set("Settings updated")
    except ValueError:
        messagebox.showerror("Invalid Input", "Please enter valid numbers for sensitivity values")

def fetch_data():
    """Fetch data from NodeMCU with retries"""
    global fps_counter, last_data_time, fps
    
    # Calculate FPS (Frames Per Second / Data updates per second)
    current_time = time.time()
    fps_counter += 1
    
    if current_time - last_data_time >= 1.0:
        fps = fps_counter
        fps_counter = 0
        last_data_time = current_time
    
    for attempt in range(MAX_RETRIES):
        try:
            response = requests.get(DATA_URL, timeout=CONNECTION_TIMEOUT)
            if response.status_code == 200:
                connection_status_var.set(f"Connected ({fps} FPS)")
                return json.loads(response.text)
            else:
                connection_status_var.set(f"Error: HTTP {response.status_code}")
                time.sleep(0.5)
        except requests.exceptions.RequestException as e:
            connection_status_var.set(f"Connection error: {type(e).__name__}")
            time.sleep(0.5)
    return None

def smooth_data(data, history_buffer):
    """Apply smoothing to sensor data"""
    history_buffer.append(data)
    return sum(history_buffer) / len(history_buffer)

def apply_acceleration(value, threshold):
    """Apply non-linear acceleration to movements for better control"""
    if abs(value) <= threshold:
        return 0  # Filter out minor movements
    
    # The further from threshold, the more acceleration applies
    sign = 1 if value > 0 else -1
    normalized = abs(value) - threshold
    return sign * (normalized ** ACCELERATION_FACTOR)

def process_sensor_data(data):
    """Process sensor data and control mouse"""
    global mouse_pos, flex_angle, flex_raw
    
    if not data or is_paused:
        return
    
    # Get accelerometer and gyroscope data
    accel = data["accelerometer"]
    gyro = data["gyroscope"]
    
    # Get flex sensor data if available
    if "flex" in data:
        flex_angle = data["flex"]["angle"]
        flex_raw = data["flex"].get("raw", 0)
        
        # Check if click is detected from NodeMCU
        if data["flex"]["click"]:
            pyautogui.click()
            click_label.config(text="CLICK!")
            root.after(500, lambda: click_label.config(text=""))
    
    # Apply calibration
    calibrated_accel = {
        "x": accel["x"] - baseline_accel["x"],
        "y": accel["y"] - baseline_accel["y"],
        "z": accel["z"] - baseline_accel["z"]
    }
    
    calibrated_gyro = {
        "x": gyro["x"] - baseline_gyro["x"],
        "y": gyro["y"] - baseline_gyro["y"],
        "z": gyro["z"] - baseline_gyro["z"]
    }
    
    # Apply smoothing to reduce jitter
    for axis in ["x", "y", "z"]:
        calibrated_accel[axis] = smooth_data(calibrated_accel[axis], accel_history[axis])
        calibrated_gyro[axis] = smooth_data(calibrated_gyro[axis], gyro_history[axis])
    
    # Calculate mouse movement based on the selected mode
    dx, dy = 0, 0
    
    if movement_mode == MODE_ABSOLUTE:
        # Absolute mode: Map tilt angle directly to screen position
        # This is similar to your original code
        dx = -calibrated_gyro["y"] if abs(calibrated_gyro["y"]) > GYRO_THRESHOLD else 0
        dy = calibrated_gyro["x"] if abs(calibrated_gyro["x"]) > GYRO_THRESHOLD else 0
        
        # Scale the movement
        dx = dx * (SCREEN_WIDTH / MOUSE_SENSITIVITY_X)
        dy = dy * (SCREEN_HEIGHT / MOUSE_SENSITIVITY_Y)
        
        # Update position
        mouse_pos["x"] += dx
        mouse_pos["y"] += dy
    
    # When MODE_RELATIVE is selected, replace the existing relative mode implementation with this:
    elif movement_mode == MODE_RELATIVE:
    # Get current gyroscope values 
        gyro_x = calibrated_gyro["x"]
        gyro_y = calibrated_gyro["y"]
    
    # Initialize movement deltas
        dx = 0
        dy = 0
    
    # Only process movement if above threshold to eliminate drift
    # This is key to keeping cursor in place when sensor is in neutral position
        if abs(gyro_y) > GYRO_THRESHOLD:
        # Invert Y axis for natural movement direction
            dx = -gyro_y
    
        if abs(gyro_x) > GYRO_THRESHOLD:
            dy = gyro_x
    
    # Apply the requested amplification factor (200.0) to make movements more responsive
    # This makes small tilts translate to meaningful cursor movement
        amplification_factor = 200.0
    
    # Apply amplification and sensitivity adjustments
        if abs(dx) > 0:
            dx_sign = 1 if dx > 0 else -1
            dx = dx_sign * ((abs(dx) * amplification_factor) / MOUSE_SENSITIVITY_X)
    
        if abs(dy) > 0:
            dy_sign = 1 if dy > 0 else -1
            dy = dy_sign * ((abs(dy) * amplification_factor) / MOUSE_SENSITIVITY_Y)
    
    # Apply exponential scaling for larger movements
    # This gives finer control for small movements while allowing fast traversal with larger tilts
        if abs(dx) > 1.0:
            dx = dx * (abs(dx) ** (ACCELERATION_FACTOR - 1.0))
    
        if abs(dy) > 1.0:
            dy = dy * (abs(dy) ** (ACCELERATION_FACTOR - 1.0))
    
    # Get current actual mouse position
        current_x, current_y = pyautogui.position()
    
    # Update position with calculated movement
        new_x = current_x + dx
        new_y = current_y + dy
    
    # Update stored mouse position
        mouse_pos["x"] = new_x
        mouse_pos["y"] = new_y
    
    else:
        # Normal movement when tilted
        dx = dx * (12.0 / MOUSE_SENSITIVITY_X)
        dy = dy * (12.0 / MOUSE_SENSITIVITY_Y)
        current_x, current_y = pyautogui.position()
        mouse_pos["x"] = current_x + dx
        mouse_pos["y"] = current_y + dy
    
    # Constrain to screen boundaries
    mouse_pos["x"] = max(0, min(SCREEN_WIDTH - 1, mouse_pos["x"]))
    mouse_pos["y"] = max(0, min(SCREEN_HEIGHT - 1, mouse_pos["y"]))
    
    # Move the mouse
    pyautogui.moveTo(mouse_pos["x"], mouse_pos["y"])
    
    # Update UI
    if show_debug:
        accel_label.config(text=f"Accel: X: {calibrated_accel['x']:.2f}, Y: {calibrated_accel['y']:.2f}, Z: {calibrated_accel['z']:.2f}")
        gyro_label.config(text=f"Gyro: X: {calibrated_gyro['x']:.2f}, Y: {calibrated_gyro['y']:.2f}, Z: {calibrated_gyro['z']:.2f}")
        mouse_label.config(text=f"Mouse: X: {mouse_pos['x']:.0f}, Y: {mouse_pos['y']:.0f}, Mode: {movement_mode}")
        flex_label.config(text=f"Flex Sensor: Angle: {flex_angle:.1f}° (Raw: {flex_raw})")

def update_ui():
    """Update UI and process data periodically"""
    if is_running:
        try:
            data = fetch_data()
            if data:
                process_sensor_data(data)
        except Exception as e:
            error_msg = str(e)
            if len(error_msg) > 50:
                error_msg = error_msg[:50] + "..."
            status_var.set(f"Error: {error_msg}")
        finally:
            root.after(UPDATE_INTERVAL, update_ui)

# Create UI elements
main_frame = ttk.Frame(root, padding="10")
main_frame.pack(fill=tk.BOTH, expand=True)

# Status frame
status_frame = ttk.LabelFrame(main_frame, text="Status")
status_frame.pack(fill=tk.X, expand=True, pady=5)

status_var = tk.StringVar(value=f"Mode: {movement_mode}")
status_label = ttk.Label(status_frame, textvariable=status_var, font=("Arial", 12, "bold"))
status_label.pack(pady=5)

connection_status_var = tk.StringVar(value="Initializing...")
connection_status_label = ttk.Label(status_frame, textvariable=connection_status_var)
connection_status_label.pack(pady=2)

click_label = ttk.Label(status_frame, text="", font=("Arial", 16, "bold"), foreground="red")
click_label.pack()

# Debug info
debug_frame = ttk.LabelFrame(main_frame, text="Sensor Data")
if show_debug:
    debug_frame.pack(fill=tk.X, expand=True, pady=5)

accel_label = ttk.Label(debug_frame, text="Accel: X: 0.00, Y: 0.00, Z: 0.00")
accel_label.pack(anchor=tk.W)

gyro_label = ttk.Label(debug_frame, text="Gyro: X: 0.00, Y: 0.00, Z: 0.00")
gyro_label.pack(anchor=tk.W)

mouse_label = ttk.Label(debug_frame, text="Mouse: X: 0, Y: 0")
mouse_label.pack(anchor=tk.W)

flex_label = ttk.Label(debug_frame, text="Flex Sensor: Angle: 0.00° (Raw: 0)")
flex_label.pack(anchor=tk.W)

# Movement mode selection
mode_frame = ttk.LabelFrame(main_frame, text="Movement Mode")
mode_frame.pack(fill=tk.X, expand=True, pady=5)

mode_var = tk.StringVar(value=movement_mode)

mode_info = {
    MODE_ABSOLUTE: "Tilt directly controls cursor position (return to neutral brings cursor to center)",
    MODE_RELATIVE: "Tilt controls cursor movement speed (return to neutral stops cursor where it is)",
    MODE_HYBRID: "Combines features of both modes for more natural control"
}

for idx, mode in enumerate([MODE_RELATIVE, MODE_ABSOLUTE, MODE_HYBRID]):
    mode_radio = ttk.Radiobutton(
        mode_frame, 
        text=mode, 
        variable=mode_var, 
        value=mode, 
        command=lambda m=mode: change_movement_mode(m)
    )
    mode_radio.grid(row=0, column=idx, padx=10, pady=5, sticky=tk.W)
    
    # Add tooltip-like description
    ttk.Label(mode_frame, text=mode_info[mode], font=("Arial", 8), foreground="gray").grid(
        row=1, column=idx, padx=10, pady=(0, 5), sticky=tk.W
    )

# Settings
settings_frame = ttk.LabelFrame(main_frame, text="Settings")
settings_frame.pack(fill=tk.X, expand=True, pady=5)

settings_grid = ttk.Frame(settings_frame)
settings_grid.pack(fill=tk.X, expand=True, padx=10, pady=5)

# X sensitivity
ttk.Label(settings_grid, text="X Sensitivity:").grid(row=0, column=0, sticky=tk.W, padx=5, pady=2)
sensitivity_x_var = tk.StringVar(value=str(MOUSE_SENSITIVITY_X))
sensitivity_x_entry = ttk.Entry(settings_grid, textvariable=sensitivity_x_var, width=8)
sensitivity_x_entry.grid(row=0, column=1, sticky=tk.W, padx=5, pady=2)
ttk.Label(settings_grid, text="(Lower = more sensitive)").grid(row=0, column=2, sticky=tk.W, padx=5, pady=2)

# Y sensitivity
ttk.Label(settings_grid, text="Y Sensitivity:").grid(row=1, column=0, sticky=tk.W, padx=5, pady=2)
sensitivity_y_var = tk.StringVar(value=str(MOUSE_SENSITIVITY_Y))
sensitivity_y_entry = ttk.Entry(settings_grid, textvariable=sensitivity_y_var, width=8)
sensitivity_y_entry.grid(row=1, column=1, sticky=tk.W, padx=5, pady=2)
ttk.Label(settings_grid, text="(Lower = more sensitive)").grid(row=1, column=2, sticky=tk.W, padx=5, pady=2)

# Gyro threshold
ttk.Label(settings_grid, text="Gyro Threshold:").grid(row=0, column=3, sticky=tk.W, padx=5, pady=2)
threshold_var = tk.StringVar(value=str(GYRO_THRESHOLD))
threshold_entry = ttk.Entry(settings_grid, textvariable=threshold_var, width=8)
threshold_entry.grid(row=0, column=4, sticky=tk.W, padx=5, pady=2)
ttk.Label(settings_grid, text="(Deadzone)").grid(row=0, column=5, sticky=tk.W, padx=5, pady=2)

# Acceleration factor
ttk.Label(settings_grid, text="Accel Factor:").grid(row=1, column=3, sticky=tk.W, padx=5, pady=2)
acceleration_var = tk.StringVar(value=str(ACCELERATION_FACTOR))
acceleration_entry = ttk.Entry(settings_grid, textvariable=acceleration_var, width=8)
acceleration_entry.grid(row=1, column=4, sticky=tk.W, padx=5, pady=2)
ttk.Label(settings_grid, text="(Higher = more acceleration)").grid(row=1, column=5, sticky=tk.W, padx=5, pady=2)

# Apply button
ttk.Button(settings_frame, text="Apply Settings", command=update_sensitivity).pack(pady=5)

# Control buttons
buttons_frame = ttk.Frame(main_frame)
buttons_frame.pack(fill=tk.X, expand=True, pady=10)

calibrate_btn = ttk.Button(buttons_frame, text="Calibrate", command=start_calibration)
calibrate_btn.pack(side=tk.LEFT, padx=5)

pause_btn = ttk.Button(buttons_frame, text="Pause", command=toggle_pause)
pause_btn.pack(side=tk.LEFT, padx=5)

debug_btn = ttk.Button(buttons_frame, text="Hide Debug" if show_debug else "Show Debug", command=toggle_debug)
debug_btn.pack(side=tk.LEFT, padx=5)

ttk.Button(buttons_frame, text="Exit", command=lambda: set_running(False)).pack(side=tk.RIGHT, padx=5)

# Connection info frame
conn_frame = ttk.LabelFrame(main_frame, text="Connection")
conn_frame.pack(fill=tk.X, expand=True, pady=5)

conn_frame_content = ttk.Frame(conn_frame)
conn_frame_content.pack(fill=tk.X, expand=True, pady=5)

ttk.Label(conn_frame_content, text="NodeMCU IP:").grid(row=0, column=0, sticky=tk.W, padx=5)
ip_var = tk.StringVar(value=NODEMCU_IP)
ip_entry = ttk.Entry(conn_frame_content, textvariable=ip_var, width=15)
ip_entry.grid(row=0, column=1, sticky=tk.W, padx=5)

def update_ip():
    global NODEMCU_IP, DATA_URL
    NODEMCU_IP = ip_var.get().strip()
    DATA_URL = f'http://{NODEMCU_IP}/data'
    connection_status_var.set(f"Connecting to {NODEMCU_IP}...")
    # Clear data history when changing connection
    for axis in ["x", "y", "z"]:
        accel_history[axis].clear()
        gyro_history[axis].clear()

ttk.Button(conn_frame_content, text="Update IP", command=update_ip).grid(row=0, column=2, padx=5)
ttk.Label(conn_frame_content, text="URL:").grid(row=1, column=0, sticky=tk.W, padx=5)
ttk.Label(conn_frame_content, text=DATA_URL).grid(row=1, column=1, columnspan=2, sticky=tk.W, padx=5)

# Keyboard shortcuts info
shortcuts_frame = ttk.LabelFrame(main_frame, text="Keyboard Shortcuts")
shortcuts_frame.pack(fill=tk.X, expand=True, pady=5)

shortcuts = """
• ESC: Emergency stop (move mouse to top-left corner)
• SPACE: Toggle pause/resume (when window is focused)
• C: Recalibrate sensor
• 1/2/3: Switch movement modes
"""
ttk.Label(shortcuts_frame, text=shortcuts, justify=tk.LEFT).pack(anchor=tk.W, pady=5)

# Bind keyboard shortcuts
def handle_key(event):
    if event.keysym == "space":
        toggle_pause()
    elif event.keysym == "c":
        start_calibration()
    elif event.keysym == "1":
        change_movement_mode(MODE_RELATIVE)
    elif event.keysym == "2":
        change_movement_mode(MODE_ABSOLUTE)
    elif event.keysym == "3":
        change_movement_mode(MODE_HYBRID)
    elif event.keysym == "Escape":
        # Emergency stop - move to corner to trigger failsafe
        pyautogui.moveTo(0, 0)
    
    # Debug message to confirm key press was detected
    print(f"Key pressed: {event.keysym}")

root.bind_all("<Key>", handle_key) 

# Function to force focus for keyboard events
def ensure_focus(event=None):
    root.focus_set()
    print("Focus set to main window")

# Bind focus event to main window and all child frames
root.bind("<FocusIn>", ensure_focus)
main_frame.bind("<Button-1>", ensure_focus)

# Call focus explicitly at startup
root.after(1500, ensure_focus)  # Set focus after a short delay

# Start the data processing
def initialize():
    global mouse_pos
    # Get current mouse position at start
    current_x, current_y = pyautogui.position()
    mouse_pos = {"x": current_x, "y": current_y}
    root.after(1000, lambda: root.focus_force())
    
    # Start updating UI after a short delay
    root.after(1000, update_ui)
    connection_status_var.set(f"Connecting to {NODEMCU_IP}...")
    
    # Show a welcome message
    messagebox.showinfo(
        "Gesture Mouse Control", 
        "Welcome to Gesture Mouse Control!\n\n"
        "• The default 'Relative' mode lets you control cursor speed with tilt angle\n"
        "• Click 'Calibrate' while holding the sensor in neutral position\n"
        "• Use the flex sensor for clicking\n"
        "• Adjust sensitivity settings if movement is too fast or slow\n\n"
        "Press OK to start"
    )

if __name__ == "__main__":
    try:
        initialize()
        root.mainloop()
    except Exception as e:
        print(f"Critical Error: {e}")
        sys.exit(1)

Credits

roboattic Lab
13 projects • 9 followers
YouTube Content Creator Robotics Enthusiast

Comments

Related channels and tags
  • Animals
  • Animals
  • Gaming
  • Gaming
  • Microsoft
  • Robotics
  • Robotics