Implement OpenCV face detection (#310)

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <parent>

        <groupId>cz.fidentis</groupId>

        <artifactId>FIDENTIS-Analyst-parent</artifactId>

        <version>master-SNAPSHOT</version>

    </parent>

    <artifactId>Detection</artifactId>

    <packaging>nbm</packaging>

    <build>

        <plugins>

            <plugin>

                <groupId>org.apache.netbeans.utilities</groupId>

                <artifactId>nbm-maven-plugin</artifactId>

                <extensions>true</extensions>

                <configuration>

                    <useOSGiDependencies>true</useOSGiDependencies>

                    <publicPackages> <!-- expose API/packages to other modules -->

                        <publicPackage>cz.fidentis.detection.*</publicPackage>

                    </publicPackages>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <configuration>

                    <source>1.8</source>

                    <target>1.8</target>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-jar-plugin</artifactId>

                <configuration>

                    <archive>

                        <manifestFile>${project.build.outputDirectory}/META-INF/MANIFEST.MF</manifestFile>

                    </archive>

                </configuration>

            </plugin>

        </plugins>

    </build>

    <dependencies>

        <dependency>

            <groupId>org.netbeans.api</groupId>

            <artifactId>org-openide-util</artifactId>

            <version>RELEASE170</version>

        </dependency>

        <dependency>

            <groupId>org.openpnp</groupId>

            <artifactId>opencv</artifactId>

            <version>4.9.0-0</version>

        </dependency>

        <dependency>

            <groupId>javax.vecmath</groupId>

            <artifactId>vecmath</artifactId>

            <version>${version.javax.vecmath}</version>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>Rendering</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>FaceData</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>FaceEngines</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>LandmarksData</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>GeometryEngines</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>GeometryData</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

    </dependencies>

    <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

    </properties>

</project>

 No newline at end of file

package cz.fidentis.detection;

import cz.fidentis.analyst.Logger;

import cz.fidentis.analyst.canvas.Canvas;

import cz.fidentis.analyst.canvas.CanvasState;

import cz.fidentis.analyst.data.face.HumanFace;

import cz.fidentis.analyst.data.landmarks.Landmark;

import cz.fidentis.analyst.data.landmarks.LandmarksFactory;

import cz.fidentis.analyst.data.landmarks.MeshVicinity;

import cz.fidentis.analyst.data.mesh.MeshTriangle;

import cz.fidentis.analyst.data.ray.Ray;

import cz.fidentis.analyst.data.ray.RayIntersection;

import cz.fidentis.analyst.drawables.Drawable;

import cz.fidentis.analyst.drawables.DrawableFace;

import cz.fidentis.analyst.engines.face.FaceStateServices;

import cz.fidentis.analyst.engines.raycasting.RayIntersectionConfig;

import cz.fidentis.analyst.engines.raycasting.RayIntersectionServices;

import cz.fidentis.analyst.engines.sampling.PointSamplingConfig;

import cz.fidentis.analyst.engines.symmetry.SymmetryConfig;

import cz.fidentis.analyst.rendering.Camera;

import cz.fidentis.analyst.rendering.RenderingMode;

import cz.fidentis.detection.models.FaceDetectionInformation;

import cz.fidentis.detection.models.SignificantPoint;

import org.openide.util.Pair;

import java.util.ArrayList;

import java.util.Collections;

import java.util.Comparator;

import java.util.List;

import java.util.Objects;

/**

 * Services for detecting faces and adding landmarks to them.

 *

 * @author Jan Popelas

 */

public class FaceDetectionServices {

    private static final double X_ANGLE_ROTATION_INCREMENT = 30;

    private static final int MINIMAL_SIGNIFICANT_POINTS = 5;

    private FaceDetectionServices() {

    }

    /**

     * Detects faces and adds landmarks to them.

     *

     * @param canvas        the canvas

     * @param primaryFace   the primary face

     * @param secondaryFace the secondary face

     * @return the pair of the primary face landmarks and the secondary face landmarks

     */

    public static List<Landmark> detectAndAddLandmarks(Canvas canvas, HumanFace primaryFace, HumanFace secondaryFace) {

        OpenCVYuNetFaceDetection faceDetector = getFaceDetector();

        CanvasState formerState = canvas.getState();

        presetCanvasForOptimalImaging(canvas);

        List<Landmark> primaryFaceLandmarks = detectLandmarksInPrimaryFace(canvas, faceDetector);

        List<Landmark> secondaryFaceLandmarks = detectLandmarksInSecondaryFace(canvas, faceDetector);

        for (Landmark primaryFaceLandmark : primaryFaceLandmarks) {

            canvas.getScene().getDrawableFeaturePoints(canvas.getScene().getPrimaryFaceSlot()).addFeaturePoint(primaryFaceLandmark);

            primaryFace.addCustomLandmark(primaryFaceLandmark);

        }

        for (Landmark secondaryFaceLandmark : secondaryFaceLandmarks) {

            canvas.getScene().getDrawableFeaturePoints(canvas.getScene().getSecondaryFaceSlot()).addFeaturePoint(secondaryFaceLandmark);

            secondaryFace.addCustomLandmark(secondaryFaceLandmark);

        }

        canvas.setState(formerState);

        primaryFaceLandmarks.addAll(secondaryFaceLandmarks);

        return primaryFaceLandmarks;

    }

    /**

     * Generates multiple cameras around the face that can be used for face detection.

     * @param camera the camera

     * @param face the face

     * @return the list of alternative cameras

     */

    private static List<Camera> generateAlternativeCameras(Camera camera, HumanFace face) {

        FaceStateServices.updateBoundingBox(face, FaceStateServices.Mode.COMPUTE_IF_ABSENT);

        FaceStateServices.updateSymmetryPlane(face, FaceStateServices.Mode.COMPUTE_IF_ABSENT, getSymmetryConfig());

        camera.zoomToFitPerpendicular(face.getBoundingBox(), face.getSymmetryPlane().getNormal());

        List<Camera> alternativeCameras = new ArrayList<>();

        for (double xAngle = 0; xAngle < 360; xAngle += X_ANGLE_ROTATION_INCREMENT) {

            Camera newCamera = camera.copy();

            newCamera.rotate(xAngle, 0);

            alternativeCameras.add(newCamera);

        }

        return alternativeCameras;

    }

    /**

     * Gets the symmetry configuration.

     * @return the symmetry configuration

     */

    private static SymmetryConfig getSymmetryConfig() {

        return new SymmetryConfig(

                SymmetryConfig.Method.ROBUST_MESH,

                new PointSamplingConfig(PointSamplingConfig.Method.UNIFORM_SPACE, 200),

                new PointSamplingConfig(PointSamplingConfig.Method.UNIFORM_SPACE, 200)

        );

    }

    /**

     * Detects landmarks in the primary face (should be full body of scan).

     *

     * @param canvas       the canvas

     * @param faceDetector the face detector

     * @return List of landmarks that were detected for a primary face

     */

    private static List<Landmark> detectLandmarksInPrimaryFace(Canvas canvas, OpenCVYuNetFaceDetection faceDetector) {

        DrawableFace primaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getPrimaryFaceSlot());

        DrawableFace secondaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getSecondaryFaceSlot());

        primaryFace.show(true);

        secondaryFace.show(false);

        List<Camera> alternativeCameras = generateAlternativeCameras(canvas.getCamera(), primaryFace.getHumanFace());

        List<Landmark> result = Collections.emptyList();

        for (Camera camera : alternativeCameras) {

            canvas.setCamera(camera);

            result = detectLandmarksFromCanvas(canvas, faceDetector);

            if (!result.isEmpty()) {

                break;

            }

        }

        secondaryFace.show(true);

        return result;

    }

    /**

     * Detects landmarks in the secondary face.

     *

     * @param canvas       the canvas

     * @param faceDetector the face detector

     * @return List of landmarks that were detected for a secondary face

     */

    private static List<Landmark> detectLandmarksInSecondaryFace(Canvas canvas, OpenCVYuNetFaceDetection faceDetector) {

        DrawableFace primaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getPrimaryFaceSlot());

        DrawableFace secondaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getSecondaryFaceSlot());

        primaryFace.show(false);

        secondaryFace.show(true);

        List<Camera> alternativeCameras = generateAlternativeCameras(canvas.getCamera(), secondaryFace.getHumanFace());

        List<Landmark> result = Collections.emptyList();

        for (Camera camera : alternativeCameras) {

            canvas.setCamera(camera);

            result = detectLandmarksFromCanvas(canvas, faceDetector);

            if (!result.isEmpty()) {

                break;

            }

        }

        primaryFace.show(true);

        return result;

    }

    /**

     * Detects landmarks from the canvas.

     *

     * @param canvas       the canvas

     * @param faceDetector the face detector

     * @return List of landmarks that were detected

     */

    private static List<Landmark> detectLandmarksFromCanvas(Canvas canvas, OpenCVYuNetFaceDetection faceDetector) {

        canvas.renderScene();

        List<FaceDetectionInformation> detectionInformation = faceDetector.detect(canvas.captureCanvas());

        if (detectionInformation.isEmpty()) {

            return new ArrayList<>();

        }

        FaceDetectionInformation faceInfo = detectionInformation.get(0);

        if (faceInfo.significantPoints().length < MINIMAL_SIGNIFICANT_POINTS) {

            return new ArrayList<>();

        }

        List<Landmark> landmarks = new ArrayList<>();

        for (SignificantPoint significantPoint : faceInfo.significantPoints()) {

            Pair<HumanFace, RayIntersection> closestFace = calculateRayIntersection(canvas, significantPoint);

            if (closestFace == null || closestFace.second() == null) {

                continue;

            }

            landmarks.add(constructLandmark(significantPoint, closestFace.second()));

        }

        return landmarks;

    }

    /**

     * Presets the canvas for optimal imaging.

     *

     * @param canvas the canvas

     */

    private static void presetCanvasForOptimalImaging(Canvas canvas) {

        DrawableFace primaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getPrimaryFaceSlot());

        DrawableFace secondaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getSecondaryFaceSlot());

        primaryFace.show(true);

        secondaryFace.show(true);

        primaryFace.setTransparency(1.0f);

        secondaryFace.setTransparency(1.0f);

        primaryFace.setRenderMode(RenderingMode.TEXTURE);

        secondaryFace.setRenderMode(RenderingMode.TEXTURE);

        canvas.getCamera().zoomToFit(canvas.getScene());

        canvas.renderScene();

    }

    /**

     * Gets the face detector.

     *

     * @return the face detector

     */

    private static OpenCVYuNetFaceDetection getFaceDetector() {

        return new OpenCVYuNetFaceDetection();

    }

    /**

     * Calculates the ray intersection for a significant point.

     *

     * @param canvas           the canvas

     * @param significantPoint the significant point

     * @return the pair of the face and the ray intersection (or null if no intersection)

     * <p>

     * Implementation borrowed from MouseClickListener

     */

    private static Pair<HumanFace, RayIntersection> calculateRayIntersection(Canvas canvas, SignificantPoint significantPoint) {

        Ray ray = canvas.getSceneRenderer().castRayThroughPixel(significantPoint.x(), significantPoint.y(), canvas.getCamera());

        Pair<HumanFace, RayIntersection> closestFace = canvas.getScene()

                .getDrawableFaces()

                .stream()

                .filter(Drawable::isShown)

                .map(DrawableFace::getHumanFace)

                .map(face -> getRayIntersection(face, ray))

                .filter(Objects::nonNull)

                .sorted(Comparator.comparingDouble(o -> o.second().getDistance()))

                .findFirst()

                .orElse(null);

        if (closestFace == null || closestFace.first() == null) {

            Logger.print("No face found for significant point " + significantPoint);

        }

        return closestFace;

    }

    /**

     * Gets the ray intersection.

     *

     * @param face the face (or body)

     * @param ray  the ray

     * @return the pair of the face and the ray intersection (or null if no intersection)

     * <p>

     * Implementation borrowed from MouseClickListener

     */

    private static Pair<HumanFace, RayIntersection> getRayIntersection(HumanFace face, Ray ray) {

        FaceStateServices.updateOctree(face, FaceStateServices.Mode.COMPUTE_IF_ABSENT);

        RayIntersection ri = RayIntersectionServices.computeClosest(

                face.getOctree(),

                new RayIntersectionConfig(ray, MeshTriangle.Smoothing.NORMAL, false));

        return (ri == null) ? null : Pair.of(face, ri);

    }

    /**

     * Constructs a landmark based on the significantPoint and the ray intersection.

     *

     * @param significantPoint

     * @param rayIntersection

     * @return the constructed landmark

     */

    private static Landmark constructLandmark(SignificantPoint significantPoint, RayIntersection rayIntersection) {

        Landmark landmark = LandmarksFactory.createFeaturePointByName(

                significantPoint.landmarkName(),

                rayIntersection.getPosition()

        );

        landmark.setMeshVicinity(new MeshVicinity(0, rayIntersection.getPosition()));

        return landmark;

    }

}

package cz.fidentis.detection;

import cz.fidentis.detection.models.BoundingBox;

import cz.fidentis.detection.models.FaceDetectionInformation;

import cz.fidentis.detection.models.SignificantPoint;

import cz.fidentis.detection.models.SignificantPointType;

import nu.pattern.OpenCV;

import org.opencv.core.CvType;

import org.opencv.core.Mat;

import org.opencv.core.Size;

import org.opencv.imgcodecs.Imgcodecs;

import org.opencv.imgproc.Imgproc;

import org.opencv.objdetect.FaceDetectorYN;

import java.awt.image.BufferedImage;

import java.awt.image.DataBufferByte;

import java.awt.image.DataBufferInt;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.InputStream;

import java.util.ArrayList;

import java.util.List;

/**

 * Face detection using the YuNet model.

 * The model is loaded from a resource file.

 * The model is used to detect faces in an image.

 * The detected faces are represented by bounding boxes and significant points.

 * The significant points are the right eye, left eye, nose, right mouth corner, and left mouth corner

 * The model is loaded from the resource file `face_detection_yunet_2023mar.onnx`.

 * Originally available at

 * <a href="https://github.com/opencv/opencv_zoo/tree/main/models/face_detection_yunet">OpenCV Zoo - YuNet</a>

 *

 * @see OpenCVYuNetFaceDetection

 * @see FaceDetectorYN

 * @see FaceDetectionInformation

 * @author Jan Popelas

 */

public class OpenCVYuNetFaceDetection {

    private FaceDetectorYN faceDetectorModel;

    private static final String MODEL_NAME = "face_detection_yunet_2023mar.onnx";

    private static final Size MODEL_RECOGNITION_SIZE = new Size(320, 320);

    private static final String MODEL_CONFIG = "";

    private static final float MODEL_SCORE_THRESHOLD = 0.80f;

    private static final float MODEL_NMS_THRESHOLD = 0.5f;

    private static final SignificantPointType[] SIGNIFICANT_POINTS = new SignificantPointType[]{

            SignificantPointType.RIGHT_EYE,

            SignificantPointType.LEFT_EYE,

            SignificantPointType.NOSE,

            SignificantPointType.RIGHT_MOUTH_CORNER,

            SignificantPointType.LEFT_MOUTH_CORNER

    };

    /**

     * Constructor.

     * Loads the model from the resource file.

     */

    public OpenCVYuNetFaceDetection() {

        OpenCV.loadLocally();

        loadModel();

    }

    /**

     * Loads the model from the resource file.

     *

     * The reason why we load it into a temporary file is that the create method of FaceDetectorYN

     * has issues with loading path from resource and throws exception "(-5: Bad argument)"

     *

     * This circumvents the issue by creating a temporary file and loading the model from it.

     * Somehow, this works. Sue me.

     *

     * @author Jan Popelas

     */

    private void loadModel() {

        InputStream is = getClass().getClassLoader().getResourceAsStream("/" + MODEL_NAME);

        File tempFile;

        try {

            tempFile = File.createTempFile("targetmodel", ".onnx");

            tempFile.deleteOnExit();

            try (FileOutputStream out = new FileOutputStream(tempFile)) {

                byte[] buffer = new byte[1024];

                int bytesRead;

                while ((bytesRead = is.read(buffer)) != -1) {

                    out.write(buffer, 0, bytesRead);

                }

            }

            String path = tempFile.getAbsolutePath();

            faceDetectorModel = FaceDetectorYN.create(

                    path,

                    MODEL_CONFIG,

                    MODEL_RECOGNITION_SIZE,

                    MODEL_SCORE_THRESHOLD,

                    MODEL_NMS_THRESHOLD

            );

        } catch (IOException e) {

            throw new RuntimeException(e);

        }

    }

    /**

     * Constructor.

     * Loads the model from the specified file.

     *

     * @param modelFile the file containing the model

     */

    public OpenCVYuNetFaceDetection(File modelFile) {

        OpenCV.loadLocally();

        faceDetectorModel = FaceDetectorYN.create(

                modelFile.getAbsolutePath(),

                MODEL_CONFIG,

                MODEL_RECOGNITION_SIZE,

                MODEL_SCORE_THRESHOLD,

                MODEL_NMS_THRESHOLD

        );

    }

    /**

     * @param imageMatrix the image matrix to detect faces in

     * @return information about the detected faces - bounding box and significant points (if available)

     */

    public List<FaceDetectionInformation> detect(Mat imageMatrix) {

        Mat image = new Mat();

        Imgproc.cvtColor(imageMatrix, image, Imgproc.COLOR_RGBA2RGB);

        faceDetectorModel.setInputSize(new Size(image.width(), image.height()));

        Mat outputDetectionData = new Mat();

        faceDetectorModel.detect(image, outputDetectionData);

        List<FaceDetectionInformation> result = new ArrayList<>();

        for (int faceIndex = 0; faceIndex < outputDetectionData.rows(); faceIndex++) {

            result.add(getInformationFromDetectionRow(outputDetectionData.row(faceIndex)));

        }

        return result;

    }

    /**

     * @param imageFile the image file to detect faces in

     * @return information about the detected faces - bounding box and significant points (if available)

     */

    public List<FaceDetectionInformation> detect(File imageFile) {

        Mat imageMatrix = Imgcodecs.imread(imageFile.getAbsolutePath());

        return detect(imageMatrix);

    }

    /**

     * @param image the image to detect faces in

     * @return information about the detected faces - bounding box and significant points (if available)

     */

    public List<FaceDetectionInformation> detect(BufferedImage image) {

        int[] pixels = ((DataBufferInt) image.getRaster().getDataBuffer()).getData();

        byte[] data = new byte[image.getWidth() * image.getHeight() * 3];

        for (int i = 0; i < pixels.length; i++) {

            data[i * 3 + 2] = (byte) ((pixels[i] >> 16) & 0xFF);

            data[i * 3 + 1] = (byte) ((pixels[i] >> 8) & 0xFF);

            data[i * 3] = (byte) (pixels[i] & 0xFF);

        }

        Mat imageMatrix = new Mat(image.getHeight(), image.getWidth(), CvType.CV_8UC3);

        imageMatrix.put(0, 0, data);

        return detect(imageMatrix);

    }

    /**

     * @param detectionRow the row of the detection data

     * @return information about the detected face - bounding box and significant points (if available)

     */

    private static FaceDetectionInformation getInformationFromDetectionRow(Mat detectionRow) {

        double[] detection = new double[(int) (detectionRow.total() * detectionRow.channels())];

        for (int j = 0; j < detectionRow.width(); j++) {

            detection[j] = detectionRow.get(0, j)[0];

        }

        int x = (int) detection[0];

        int y = (int) detection[1];

        int width = (int) detection[2] - x;

        int height = (int) detection[3] - y;

        BoundingBox boundingBox = new BoundingBox(x, y, width, height);

        SignificantPoint[] significantPoints = new SignificantPoint[5];

        for (int j = 4; j < detection.length && j < 14; j += 2) {

            int pointIndex = (j - 4) / 2;

            SignificantPoint point = new SignificantPoint(SIGNIFICANT_POINTS[pointIndex], (int) detection[j], (int) detection[j + 1]);

            significantPoints[pointIndex] = point;

        }

        return new FaceDetectionInformation(boundingBox, significantPoints);

    }

}

package cz.fidentis.detection.models;

/**

 * Represents a bounding box in an image.

 *

 * @param x

 * @param y

 * @param width

 * @param height

 *

 * @author Jan Popelas

 */

public record BoundingBox(int x, int y, int width, int height) {

    /**

     * Returns the x coordinate of the opposite corner of the bounding box.

     * @return the x coordinate of the opposite corner of the bounding box

     */

    public int getOppositeX() {

        return x + width;

    }

    /**

     * Returns the y coordinate of the opposite corner of the bounding box.

     * @return the y coordinate of the opposite corner of the bounding box

     */

    public int getOppositeY() {

        return y + height;

    }

}

package cz.fidentis.detection.models;

/**

 * Represents information about a face detected in an image.

 *

 * @param boundingBox the bounding box of the detected face

 * @param significantPoints the significant points of the detected face

 *

 * @author Jan Popelas

 */

public record FaceDetectionInformation(BoundingBox boundingBox, SignificantPoint[] significantPoints) {

    /**

     * Constructor.

     * @param boundingBox the bounding box of the detected face

     * @param significantPoints the significant points of the detected face

     */

    public FaceDetectionInformation {

        if (boundingBox == null) {

            throw new IllegalArgumentException("Bounding box cannot be null");

        }

        if (significantPoints == null) {

            significantPoints = new SignificantPoint[0];

        }

    }

}