hand_gesturer_ecognition/gesture_recognition.py

import warnings
import threading
import cv2
import mediapipe as mp
import numpy as np
from tensorflow.keras.models import load_model
import os

# 设置环境变量以关闭oneDNN自定义操作
os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'

# 忽略SymbolDatabase.GetPrototype()弃用警告
warnings.filterwarnings("ignore", category=UserWarning, message='SymbolDatabase.GetPrototype() is deprecated')

# 初始化全局变量
hands = None
mp_draw = mp.solutions.drawing_utils
cap = None
keep_running = False
paused = False

# 模型路径和加载
model_path = 'D:/hand/hand_gesture_model.h5'
model = load_model(model_path)

# 手势类别
gesture_classes = ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09']

def start_recognition(callback=None):
    """开始手势识别"""
    global keep_running, cap, hands
    # 打开摄像头
    if cap is None or not cap.isOpened():
        cap = cv2.VideoCapture(0)
    # 初始化Hand对象
    if hands is None:
        hands = mp.solutions.hands.Hands(static_image_mode=False, max_num_hands=2,
                                         model_complexity=1, min_detection_confidence=0.5,
                                         min_tracking_confidence=0.5)
    keep_running = True
    # 启动识别线程
    threading.Thread(target=run_recognition, args=(callback,)).start()

def run_recognition(callback=None):
    """运行手势识别"""
    global keep_running, paused

    while keep_running and cap.isOpened():
        ret, img = cap.read()
        if not ret:
            break

        img = cv2.flip(img, 1)  # 翻转图像
        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # 转换颜色空间

        if not paused:
            results = hands.process(img_rgb)
            total_raised_fingers = 0

            if results.multi_hand_landmarks:
                for handLms in results.multi_hand_landmarks:
                    mp_draw.draw_landmarks(img_rgb, handLms, mp.solutions.hands.HAND_CONNECTIONS)
                    gesture, raised_fingers = detect_gesture_and_fingers(handLms)
                    total_raised_fingers += raised_fingers

            cv2.putText(img_rgb, f'Total Raised Fingers: {total_raised_fingers}', (10, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 2, cv2.LINE_AA)

            if total_raised_fingers == 5:  # 如果检测到五根手指，调用回调函数传递手指数
                if callback:
                    callback(total_raised_fingers)

        if callback:
            callback(img_rgb)  # 调用回调函数传递图像数据

def stop_recognition():
    """停止手势识别"""
    global keep_running, cap, paused
    keep_running = False
    paused = False
    if cap is not None and cap.isOpened():
        cap.release()
        cap = None
    cv2.destroyAllWindows()

def release_camera():
    """释放摄像头资源"""
    global cap
    if cap is not None and cap.isOpened():
        cap.release()
        cap = None

def detect_gesture_and_fingers(hand_landmarks):
    """检测手势和手指状态"""
    gesture_image = get_hand_image(hand_landmarks)
    gesture = predict_gesture(gesture_image)
    raised_fingers = count_raised_fingers(hand_landmarks)
    return gesture, raised_fingers

def get_hand_image(hand_landmarks):
    """获取手部图像"""
    img = np.zeros((150, 150, 3), dtype=np.uint8)
    return img

def predict_gesture(img):
    """预测手势"""
    img = cv2.resize(img, (150, 150))
    img_array = np.expand_dims(img, axis=0) / 255.0
    predictions = model.predict(img_array)
    predicted_class = gesture_classes[np.argmax(predictions)]
    return predicted_class

def count_raised_fingers(hand_landmarks):
    """计算竖起的手指数量"""
    fingers_status = [0, 0, 0, 0, 0]

    thumb_tip = hand_landmarks.landmark[mp.solutions.hands.HandLandmark.THUMB_TIP]
    thumb_ip = hand_landmarks.landmark[mp.solutions.hands.HandLandmark.THUMB_IP]
    thumb_mcp = hand_landmarks.landmark[mp.solutions.hands.HandLandmark.THUMB_MCP]
    thumb_cmc = hand_landmarks.landmark[mp.solutions.hands.HandLandmark.THUMB_CMC]

    angle_thumb = calculate_angle(thumb_cmc, thumb_mcp, thumb_tip)
    if angle_thumb > 160:
        fingers_status[0] = 1

    for i, finger_tip_id in enumerate([mp.solutions.hands.HandLandmark.INDEX_FINGER_TIP,
                                       mp.solutions.hands.HandLandmark.MIDDLE_FINGER_TIP,
                                       mp.solutions.hands.HandLandmark.RING_FINGER_TIP,
                                       mp.solutions.hands.HandLandmark.PINKY_TIP]):
        finger_tip = hand_landmarks.landmark[finger_tip_id]
        finger_pip = hand_landmarks.landmark[finger_tip_id - 2]
        finger_mcp = hand_landmarks.landmark[finger_tip_id - 3]

        angle_finger = calculate_angle(finger_mcp, finger_pip, finger_tip)
        if angle_finger > 160:
            fingers_status[i + 1] = 1

    return sum(fingers_status)

def calculate_angle(point1, point2, point3):
    """计算三个点之间的角度"""
    angle = np.arctan2(point3.y - point2.y, point3.x - point2.x) - np.arctan2(point1.y - point2.y, point1.x - point2.x)
    angle = np.abs(angle)
    if angle > np.pi:
        angle = 2 * np.pi - angle
    return angle * 180 / np.pi