Create Camera function (Windows)

Import necessary package.

import imp
from pickle import NONE
import cv2
import mediapipe as mp
import sys
import os
import json
import time
import datetime
from mediapipe.framework.formats import landmark_pb2
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
import mqtt
from mqtt import *

Create a class

To start custom function, first we need to create a class. This class will includes the initializetion, all the necessary function and the main run function.

class Camera:
	def __init__(self):

Data logging

Set the function to save the data in a json file.

def data_log(self,data):
    # file name
    now = self.log_dir.split("/")[-1]
    name = now + '_hand_track.json'
    key = str(datetime.datetime.now())

    self.log[key] = data
                
    path = os.path.join(self.log_dir, name)
    with open(path, "w") as file:
        json.dump(self.log, file, indent=4)

Video record

To save the video, we need to use the cv2.VideoWriter() function.

def vedio_log(self, cap):
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    fps = 18
    frame_size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)))
    now = self.log_dir.split("/")[-1]
    name = now + '_hand_track.avi'
    path = os.path.join(self.log_dir, name)
    out = cv2.VideoWriter(path, fourcc, fps, frame_size)
    return out

Add custom annotation

To add custom annotation, we need to use the cv2.putText() function.

def custom_annotations(self,frame):
    font = cv2.FONT_HERSHEY_SIMPLEX
    font_scale = 0.8
    color = (255, 255, 255)
    thickness = 2
    x = 10
    y = frame.shape[0] - 30
    item_list = list(self.log.items())
    if len(item_list) != 0:
        id, value = item_list[-1]
        key_text = f"{id}:{value}"
        cv2.putText(frame, key_text, (x, y), font, font_scale, color, thickness)
    return frame

Main run function

To start the hand gesture recognition, we need to create a main run function and start camera feed.

First, we need to prepare the log directory.

def run(self):
    current = os.path.abspath(__file__)
    self.log_dir = current + "logs/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    if not os.path.exists(self.log_dir):
        os.makedirs(self.log_dir)

Then, set the mqtt connection.

mqtt.setup(CLIENT,TOPIC,PREVIOUSCHECK)
mqtt.setup_emqx(CLIENT,TOPIC,PREVIOUSCHECK)

Load gesture recognition model.

with open(self.MODEL_PATH, 'rb') as f:
    model = f.read()
base_options = python.BaseOptions(model_asset_buffer=model)
    options = vision.GestureRecognizerOptions(
        base_options=base_options,
        running_mode=vision.RunningMode.LIVE_STREAM,
        num_hands=self.NUM_HANDS,
        min_hand_detection_confidence=self.MIN_HAND_DETECTION_CONFIDENCE,
        min_hand_presence_confidence=self.MIN_HAND_PRESENCE_CONFIDENCE,
        min_tracking_confidence=self.MIN_TRACKING_CONFIDENCE,
        result_callback=self.save_result)
recognizer = vision.GestureRecognizer.create_from_options(options)

Set camera and video record.

cap = cv2.VideoCapture(self.CAMERA_ID)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.FRAME_WIDTH)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.FRAME_HEIGHT)
out = self.vedio_log(cap)

Turn on camera and start recognition.

 while cap.isOpened():
    success, image = cap.read()
    if not success:
        sys.exit('ERROR: Unable to read from webcam. Please verify your webcam settings.')

    image = cv2.flip(image, 1)
    rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_image)
    recognizer.recognize_async(mp_image, time.time_ns() // 1_000_000)

Show FPS and custom annotation.

fps_text = 'FPS = {:.1f}'.format(self.FPS)
    text_location = (self.left_margin, self.row_size)
    current_frame = image
    cv2.putText(current_frame, fps_text, text_location, cv2.FONT_HERSHEY_DUPLEX,
                self.font_size, self.text_color, self.font_thickness, cv2.LINE_AA)
        
    current_frame = self.custom_annotations(current_frame)

Check recognition result, save the data, and add draw landmark on the hand.

if self.recognition_result_list:
    #_______________________________
    for hand_index, hand_landmarks in enumerate(
    self.recognition_result_list[0].hand_landmarks):
        # Calculate the bounding box of the hand
        x_min = min([landmark.x for landmark in hand_landmarks])
        y_min = min([landmark.y for landmark in hand_landmarks])
        y_max = max([landmark.y for landmark in hand_landmarks])

        # Convert normalized coordinates to pixel values
        frame_height, frame_width = current_frame.shape[:2]
        x_min_px = int(x_min * frame_width)
        y_min_px = int(y_min * frame_height)
        y_max_px = int(y_max * frame_height)

        # Get gesture classification results
        if self.recognition_result_list[0].gestures:
            gesture = self.recognition_result_list[0].gestures[hand_index]
            category_name = gesture[0].category_name
            score = round(gesture[0].score, 2)
            result_text = f'{category_name} ({score})'
                    
            #log values
            self.data_log(category_name)

            #MQTT Publish
            #mqtt.publish(category_name,CLIENT,TOPIC)
                    
            # Compute text size
            text_size = \
            cv2.getTextSize(result_text, cv2.FONT_HERSHEY_DUPLEX, self.label_font_size,
                            self.label_thickness)[0]
            text_width, text_height = text_size

            # Calculate text position (above the hand)
            text_x = x_min_px
            text_y = y_min_px - 10  # Adjust this value as needed

            # Make sure the text is within the frame boundaries
            if text_y < 0:
                text_y = y_max_px + text_height

            # Draw the text
            cv2.putText(current_frame, result_text, (text_x, text_y),
                        cv2.FONT_HERSHEY_DUPLEX, self.label_font_size,
                        self.label_text_color, self.label_thickness, cv2.LINE_AA)

        # Draw hand landmarks on the frame
        hand_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
        hand_landmarks_proto.landmark.extend([
        landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y,
                                        z=landmark.z) for landmark in
        hand_landmarks
        ])
        self.mp_drawing.draw_landmarks(
            current_frame,
            hand_landmarks_proto,
            self.mp_hands.HAND_CONNECTIONS,
            self.mp_drawing_styles.get_default_hand_landmarks_style(),
            self.mp_drawing_styles.get_default_hand_connections_style())

Save the video and clear recognition result.

recognition_frame = current_frame
self.recognition_result_list.clear()
out.write(current_frame)

Encode the frame as JPEG for web page response

ret, buffer = cv2.imencode('.jpg', image)
frame_jpeg = buffer.tobytes()
yield (b'--frame\r\n'
    b'Content-Type: image/jpeg\r\n\r\n' + frame_jpeg + b'\r\n')

Set keyboard break.

if cv2.waitKey(1) & 0xFF == 27:
    break

Stop the camera and release.

recognizer.close()
self.CLIENT.disconnect()
cap.release()
cv2.destroyAllWindows()