손으로 마우스 조종 with Mediapipe
이 글은 기존 블로그의 원문에서 가져왔습니다. 원문 작성일: 2024-01-10
Mediapipe
딥러닝을 이용한 라이브러리 중에 영상 정보를 통한 다양한 기능을 가진 Mediapipe가 있다.
이것을 통해서 카메라를 통해 허공에서 손을 움직여 마우스 조종을 할 수 있게 만들어 봤다.
import mediapipe as mp
import cv2
import numpy as np
import math
import pyautogui
import time
#return distance of (x1, y1), (x2, y2)
def dist_func(x1, y1, x2, y2):
return math.sqrt(math.pow(x1-x2, 2)+math.pow(y1-y2, 2))
#return mid point of (x1, y1), (x2, y2)
def mid_point(x1, y1, x2, y2):
return (x1+x2)/2., (y1+y2)/2.
#update mouse position
def update_mouse_pos(befMousePos, befPointPos, curPointPos, time, sen = 1000):
x = befMousePos[0] + (curPointPos[0] - befPointPos[0])*sen
y = befMousePos[1] + (curPointPos[1] - befPointPos[1])*sen
pyautogui.moveTo(x, y, time)
#return True, if pinch gesture
def pinch_gesture(landmark):
fingers = [False for i in range(5)]
for i in range(1, 5):
dist_a = dist_func(landmark[i*4+4].x, landmark[i*4+4].y, landmark[0].x, landmark[0].y)
dist_b = dist_func(landmark[i*4+2].x, landmark[i*4+2].y, landmark[0].x, landmark[0].y)
if dist_a < dist_b:
fingers[i] = True
if not fingers[1] and fingers[2] and fingers[3] and fingers[4]:
return True
else:
return False
mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands
camera = cv2.VideoCapture(1)
mousePos = pyautogui.position()
befMousePos = [mousePos.x, mousePos.y]
befPointPos = [0.5, 0.5]
befTime = time.time()
befHandResult = False
with mp_hands.Hands(
max_num_hands=1,
min_detection_confidence=0.5,
min_tracking_confidence=0.5) as hands:
while camera.isOpened():
success, image = camera.read()
if not success:
continue
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
results = hands.process(image)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
thumb = hand_landmarks.landmark[4]
index = hand_landmarks.landmark[8]
if pinch_gesture(hand_landmarks.landmark):
#move mouse part
curPointPos = mid_point(thumb.x, thumb.y, index.x, index.y)
update_mouse_pos(befMousePos, befPointPos, curPointPos, time.time()-befTime)
#click part
distTI = dist_func(thumb.x, thumb.y, index.x, index.y)
if distTI < 0.05:
pyautogui.click()
befHandResult = True
befPointPos = mid_point(thumb.x, thumb.y, index.x, index.y)
mp_drawing.draw_landmarks(
image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
else:
befHandResult = False
befTime=time.time()
mousePos = pyautogui.position()
befMousePos = [mousePos.x, mousePos.y]
#cv2.imshow('image', image)
#if cv2.waitKey(1) == ord('q'):
# break
집게 손 모양을 할 때에만 마우스 포인터의 위치를 조정할 수 있고 무언갈 집는 손동작으로 클릭 기능을 구현했다.