Emotion Detection with Python: A Flask API for Static Images and Real-Time Video

pip install Flask fer opencv-python-headless numpy

Flask
fer
opencv-python-headless
numpy
tensorflow
opencv-contrib-python
ffmpeg
moviepy

python3 -m venv path/to/venv

source path/to/venv/bin/activate

pip install -r requirements.txt

import os
os.environ["IMAGEIO_FFMPEG_EXE"] = "/usr/bin/ffmpeg"
from flask import Flask, request, jsonify, Response # Import Flask for creating API endpoints
from fer import FER  # Import FER for emotion detection
import cv2  # OpenCV for image and video processing
import numpy as np  # NumPy for handling image arrays

app = Flask(__name__)
detector = FER()  # Initialize the FER emotion detector

# Load OpenCV's pre-trained face detection model (Haar Cascade)
face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')

# API to process and detect emotions in static images
@app.route('/upload', methods=['POST'])
def upload_image():
    # Check if a file is provided in the request
    if 'file' not in request.files:
        return jsonify({'error': 'No file provided'}), 400

    file = request.files['file']  # Get the file from the request
    if file.filename == '':  # Check if a file was actually selected
        return jsonify({'error': 'No selected file'}), 400

    # Read the image bytes and convert it to a NumPy array
    img_bytes = file.read()
    np_img = np.frombuffer(img_bytes, np.uint8)
    image = cv2.imdecode(np_img, cv2.IMREAD_COLOR)  # Decode the image using OpenCV

    if image is None:  # If image decoding fails
        return jsonify({'error': 'Invalid image'}), 400

    # Use the FER detector to detect the top emotion in the image
    emotion, score = detector.top_emotion(image)

    # Return the detected emotion and confidence score as JSON
    return jsonify({'emotion': emotion, 'score': score})

# API to stream video from the default camera and perform real-time emotion detection
@app.route('/video_feed')
def video_feed():
    # Function to generate video frames with emotion detection
    def generate_frames():
        cap = cv2.VideoCapture(0)  # Open the default camera (0)

        while True:
            success, frame = cap.read()  # Read a frame from the camera
            if not success:  # If the camera cannot provide frames, break the loop
                break

            # Convert the frame to grayscale for face detection
            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            faces = face_cascade.detectMultiScale(
                gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30)
            )  # Detect faces in the grayscale image

            # Detect the top emotion for the current frame
            emotion, score = detector.top_emotion(frame)

            # For each detected face, draw a rectangle and annotate the emotion
            for (x, y, w, h) in faces:
                # Draw a rectangle around the face
                cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
                # Annotate the detected emotion on the frame
                cv2.putText(
                    frame, f'Emotion: {emotion}', (x, y - 10), 
                    cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0), 2
                )

            # Encode the frame to a JPEG image format
            ret, buffer = cv2.imencode('.jpg', frame)
            frame = buffer.tobytes()

            # Yield the frame in multipart format to display in the browser
            yield (b'--frame\r\n'
                   b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')

        # Release the video capture resource when done
        cap.release()

    # Stream the video frames using HTTP multipart format
    return Response(generate_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')

# Start the Flask app on the specified port and IP address
if __name__ == '__main__':
    app.run(host='0.0.0.0', port=3000)

# ### Summary:

# 1. **Static Image Processing (`/upload` endpoint)**:
#    - Accepts an image file via POST request.
#    - The image is decoded using OpenCV and processed by the FER (Facial Expression Recognition) model to detect the top emotion.
#    - Returns the emotion and confidence score in JSON format.

# 2. **Real-Time Video Stream (`/video_feed` endpoint)**:
#    - Streams video from the default camera in real-time.
#    - Detects faces in each video frame using OpenCV's Haar Cascade classifier.
#    - FER detects the top emotion for each frame.
#    - Draws a rectangle around each detected face and annotates the emotion.
#    - The video stream is sent in HTTP multipart format to be viewed in real-time in a browser.

# Both endpoints showcase emotion detection using FER and OpenCV, with one handling static images and the other real-time video.

python3 server.py

import axios from "axios";

const UPLOAD_API = "http://xxx.xxx.x.x:3000/upload" 
export const uploadImage = async (local_uri: string) => {
    try {
        const formData = new FormData();
        formData.append('file', {
            uri: local_uri,
            name: 'file',
        });

        const res = await axios.post(UPLOAD_API, formData, {
            headers: {
                'Content-Type': 'multipart/form-data',
            },
        });
        return res.data;
    } catch (error) {
        console.log(error);
        throw error
    }
}