Commit e6a1d1a3 authored by Radek Ošlejšek's avatar Radek Ošlejšek
Browse files

Merge branch '321-refactor-the-detection-module' into 'master' 

Resolve "Refactor the Detection module"

Closes #321

See merge request grp-fidentis/analyst2!347
parents a213e90d 4e86d2e0

Detection/pom.xml

deleted100644 → 0
+0 −99
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <parent>

        <groupId>cz.fidentis</groupId>

        <artifactId>FIDENTIS-Analyst-parent</artifactId>

        <version>master-SNAPSHOT</version>

    </parent>

    <artifactId>Detection</artifactId>

    <packaging>nbm</packaging>

    <build>

        <plugins>

            <plugin>

                <groupId>org.apache.netbeans.utilities</groupId>

                <artifactId>nbm-maven-plugin</artifactId>

                <extensions>true</extensions>

                <configuration>

                    <useOSGiDependencies>true</useOSGiDependencies>

                    <publicPackages> <!-- expose API/packages to other modules -->

                        <publicPackage>cz.fidentis.detection.*</publicPackage>

                    </publicPackages>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <configuration>

                    <source>1.8</source>

                    <target>1.8</target>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-jar-plugin</artifactId>

                <configuration>

                    <archive>

                        <manifestFile>${project.build.outputDirectory}/META-INF/MANIFEST.MF</manifestFile>

                    </archive>

                </configuration>

            </plugin>

        </plugins>

    </build>

    <dependencies>

        <dependency>

            <groupId>org.netbeans.api</groupId>

            <artifactId>org-openide-util</artifactId>

            <version>RELEASE170</version>

        </dependency>

        <dependency>

            <groupId>org.openpnp</groupId>

            <artifactId>opencv</artifactId>

            <version>4.9.0-0</version>

        </dependency>

        <dependency>

            <groupId>javax.vecmath</groupId>

            <artifactId>vecmath</artifactId>

            <version>${version.javax.vecmath}</version>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>Rendering</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>FaceData</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>FaceEngines</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>LandmarksData</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>GeometryEngines</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

        <dependency>

            <groupId>cz.fidentis</groupId>

            <artifactId>GeometryData</artifactId>

            <version>master-SNAPSHOT</version>

            <scope>compile</scope>

        </dependency>

    </dependencies>

    <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

    </properties>

</project>
 
 No newline at end of file

Detection/src/main/java/cz/fidentis/detection/FaceDetectionServices.java

deleted100644 → 0
+0 −305
Original line number Diff line number Diff line 
package cz.fidentis.detection;



import cz.fidentis.analyst.Logger;

import cz.fidentis.analyst.canvas.Canvas;

import cz.fidentis.analyst.canvas.CanvasState;

import cz.fidentis.analyst.data.face.HumanFace;

import cz.fidentis.analyst.data.face.HumanFaceState;

import cz.fidentis.analyst.data.landmarks.Landmark;

import cz.fidentis.analyst.data.landmarks.LandmarksFactory;

import cz.fidentis.analyst.data.landmarks.MeshVicinity;

import cz.fidentis.analyst.data.mesh.MeshTriangle;

import cz.fidentis.analyst.data.ray.Ray;

import cz.fidentis.analyst.data.ray.RayIntersection;

import cz.fidentis.analyst.drawables.Drawable;

import cz.fidentis.analyst.drawables.DrawableFace;

import cz.fidentis.analyst.engines.face.FaceStateServices;

import cz.fidentis.analyst.engines.raycasting.RayIntersectionConfig;

import cz.fidentis.analyst.engines.raycasting.RayIntersectionServices;

import cz.fidentis.analyst.engines.sampling.PointSamplingConfig;

import cz.fidentis.analyst.engines.symmetry.SymmetryConfig;

import cz.fidentis.analyst.rendering.Camera;

import cz.fidentis.analyst.rendering.RenderingMode;

import cz.fidentis.detection.models.FaceDetectionInformation;

import cz.fidentis.detection.models.SignificantPoint;

import org.openide.util.Pair;



import java.util.ArrayList;

import java.util.Collections;

import java.util.Comparator;

import java.util.List;

import java.util.Objects;





/**

 * Services for detecting faces and adding landmarks to them.

 *

 * @author Jan Popelas

 */

public class FaceDetectionServices {



    private static final double X_ANGLE_ROTATION_INCREMENT = 30;

    private static final int MINIMAL_SIGNIFICANT_POINTS = 5;



    private FaceDetectionServices() {

    }



    /**

     * Detects faces and adds landmarks to them.

     *

     * @param canvas        the canvas

     * @param primaryFace   the primary face

     * @param secondaryFace the secondary face

     * @return the pair of the primary face landmarks and the secondary face landmarks

     */

    public static List<Landmark> detectAndAddLandmarks(Canvas canvas, HumanFace primaryFace, HumanFace secondaryFace) {

        OpenCVYuNetFaceDetection faceDetector = getFaceDetector();

        CanvasState formerState = canvas.getState();

        HumanFaceState primaryFaceState = primaryFace.getState();

        HumanFaceState secondaryFaceState = secondaryFace.getState();



        List<Landmark> primaryFaceLandmarks = detectLandmarksInPrimaryFace(canvas, faceDetector);

        List<Landmark> secondaryFaceLandmarks = detectLandmarksInSecondaryFace(canvas, faceDetector);



        for (Landmark primaryFaceLandmark : primaryFaceLandmarks) {

            canvas.getScene().getDrawableFeaturePoints(canvas.getScene().getPrimaryFaceSlot()).addFeaturePoint(primaryFaceLandmark);

            primaryFace.addCustomLandmark(primaryFaceLandmark);

        }



        for (Landmark secondaryFaceLandmark : secondaryFaceLandmarks) {

            canvas.getScene().getDrawableFeaturePoints(canvas.getScene().getSecondaryFaceSlot()).addFeaturePoint(secondaryFaceLandmark);

            secondaryFace.addCustomLandmark(secondaryFaceLandmark);

        }



        canvas.setState(formerState);

        primaryFace.setSymmetryPlane(primaryFaceState.symmetryPlane());

        secondaryFace.setSymmetryPlane(secondaryFaceState.symmetryPlane());



        primaryFaceLandmarks.addAll(secondaryFaceLandmarks);

        return primaryFaceLandmarks;

    }



    /**

     * Generates multiple cameras around the face that can be used for face detection.

     * @param camera the camera

     * @param face the face

     * @return the list of alternative cameras

     */

    private static List<Camera> generateAlternativeCameras(Camera camera, HumanFace face) {

        FaceStateServices.updateBoundingBox(face, FaceStateServices.Mode.COMPUTE_IF_ABSENT);

        FaceStateServices.updateSymmetryPlane(face, FaceStateServices.Mode.COMPUTE_IF_ABSENT, getSymmetryConfig());

        camera.zoomToFitPerpendicular(face.getBoundingBox(), face.getSymmetryPlane().getNormal());





        List<Camera> alternativeCameras = new ArrayList<>();

        for (double xAngle = 0; xAngle < 360; xAngle += X_ANGLE_ROTATION_INCREMENT) {

            Camera newCamera = camera.copy();

            newCamera.rotate(xAngle, 0);

            alternativeCameras.add(newCamera);

        }



        return alternativeCameras;

    }



    /**

     * Gets the symmetry configuration.

     * @return the symmetry configuration

     */

    private static SymmetryConfig getSymmetryConfig() {

        return new SymmetryConfig(

                SymmetryConfig.Method.ROBUST_MESH,

                new PointSamplingConfig(PointSamplingConfig.Method.UNIFORM_SPACE, 200),

                new PointSamplingConfig(PointSamplingConfig.Method.UNIFORM_SPACE, 200)

        );



    }



    /**

     * Detects landmarks in the primary face (should be full body of scan).

     *

     * @param canvas       the canvas

     * @param faceDetector the face detector

     * @return List of landmarks that were detected for a primary face

     */

    private static List<Landmark> detectLandmarksInPrimaryFace(Canvas canvas, OpenCVYuNetFaceDetection faceDetector) {

        DrawableFace primaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getPrimaryFaceSlot());

        DrawableFace secondaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getSecondaryFaceSlot());

        presetCanvasForOptimalImaging(canvas);

        primaryFace.show(true);

        secondaryFace.show(false);



        List<Camera> alternativeCameras = generateAlternativeCameras(canvas.getCamera(), primaryFace.getHumanFace());



        List<Landmark> result = Collections.emptyList();



        for (Camera camera : alternativeCameras) {

            canvas.setCamera(camera);

            result = detectLandmarksFromCanvas(canvas, faceDetector);

            if (!result.isEmpty()) {

                break;

            }

        }



        secondaryFace.show(true);

        return result;

    }



    /**

     * Detects landmarks in the secondary face.

     *

     * @param canvas       the canvas

     * @param faceDetector the face detector

     * @return List of landmarks that were detected for a secondary face

     */

    private static List<Landmark> detectLandmarksInSecondaryFace(Canvas canvas, OpenCVYuNetFaceDetection faceDetector) {

        DrawableFace primaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getPrimaryFaceSlot());

        DrawableFace secondaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getSecondaryFaceSlot());

        presetCanvasForOptimalImaging(canvas);

        primaryFace.show(false);

        secondaryFace.show(true);



        List<Camera> alternativeCameras = generateAlternativeCameras(canvas.getCamera(), secondaryFace.getHumanFace());



        List<Landmark> result = Collections.emptyList();



        for (Camera camera : alternativeCameras) {

            canvas.setCamera(camera);

            result = detectLandmarksFromCanvas(canvas, faceDetector);

            if (!result.isEmpty()) {

                break;

            }

        }



        primaryFace.show(true);

        return result;

    }



    /**

     * Detects landmarks from the canvas.

     *

     * @param canvas       the canvas

     * @param faceDetector the face detector

     * @return List of landmarks that were detected

     */

    private static List<Landmark> detectLandmarksFromCanvas(Canvas canvas, OpenCVYuNetFaceDetection faceDetector) {

        canvas.renderScene();

        List<FaceDetectionInformation> detectionInformation = faceDetector.detect(canvas.captureCanvas());



        if (detectionInformation.isEmpty()) {

            return new ArrayList<>();

        }



        FaceDetectionInformation faceInfo = detectionInformation.get(0);

        if (faceInfo.significantPoints().length < MINIMAL_SIGNIFICANT_POINTS) {

            return new ArrayList<>();

        }



        List<Landmark> landmarks = new ArrayList<>();

        for (SignificantPoint significantPoint : faceInfo.significantPoints()) {

            Pair<HumanFace, RayIntersection> closestFace = calculateRayIntersection(canvas, significantPoint);

            if (closestFace == null || closestFace.second() == null) {

                continue;

            }

            landmarks.add(constructLandmark(significantPoint, closestFace.second()));

        }



        return landmarks;

    }



    /**

     * Presets the canvas for optimal imaging.

     *

     * @param canvas the canvas

     */

    private static void presetCanvasForOptimalImaging(Canvas canvas) {

        DrawableFace primaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getPrimaryFaceSlot());

        DrawableFace secondaryFace = canvas.getScene().getDrawableFace(canvas.getScene().getSecondaryFaceSlot());



        canvas.setDarkBackground(false);



        primaryFace.show(true);

        secondaryFace.show(true);



        primaryFace.setTransparency(1.0f);

        secondaryFace.setTransparency(1.0f);



        primaryFace.setRenderMode(RenderingMode.TEXTURE);

        secondaryFace.setRenderMode(RenderingMode.TEXTURE);



        canvas.getCamera().zoomToFit(canvas.getScene());



        canvas.renderScene();

    }



    /**

     * Gets the face detector.

     *

     * @return the face detector

     */

    private static OpenCVYuNetFaceDetection getFaceDetector() {

        return new OpenCVYuNetFaceDetection();

    }



    /**

     * Calculates the ray intersection for a significant point.

     *

     * @param canvas           the canvas

     * @param significantPoint the significant point

     * @return the pair of the face and the ray intersection (or null if no intersection)

     * <p>

     * Implementation borrowed from MouseClickListener

     */

    private static Pair<HumanFace, RayIntersection> calculateRayIntersection(Canvas canvas, SignificantPoint significantPoint) {

        Ray ray = canvas.getSceneRenderer().castRayThroughPixel(significantPoint.x(), significantPoint.y(), canvas.getCamera());



        Pair<HumanFace, RayIntersection> closestFace = canvas.getScene()

                .getDrawableFaces()

                .stream()

                .filter(Drawable::isShown)

                .map(DrawableFace::getHumanFace)

                .map(face -> getRayIntersection(face, ray))

                .filter(Objects::nonNull)

                .sorted(Comparator.comparingDouble(o -> o.second().getDistance()))

                .findFirst()

                .orElse(null);



        if (closestFace == null || closestFace.first() == null) {

            Logger.print("No face found for significant point " + significantPoint);

        }



        return closestFace;

    }



    /**

     * Gets the ray intersection.

     *

     * @param face the face (or body)

     * @param ray  the ray

     * @return the pair of the face and the ray intersection (or null if no intersection)

     * <p>

     * Implementation borrowed from MouseClickListener

     */

    private static Pair<HumanFace, RayIntersection> getRayIntersection(HumanFace face, Ray ray) {

        FaceStateServices.updateOctree(face, FaceStateServices.Mode.COMPUTE_IF_ABSENT);

        RayIntersection ri = RayIntersectionServices.computeClosest(

                face.getOctree(),

                new RayIntersectionConfig(ray, MeshTriangle.Smoothing.NORMAL, false));

        return (ri == null) ? null : Pair.of(face, ri);

    }



    /**

     * Constructs a landmark based on the significantPoint and the ray intersection.

     *

     * @param significantPoint

     * @param rayIntersection

     * @return the constructed landmark

     */

    private static Landmark constructLandmark(SignificantPoint significantPoint, RayIntersection rayIntersection) {

        Landmark landmark = LandmarksFactory.createFeaturePointByName(

                significantPoint.landmarkName(),

                rayIntersection.getPosition()

        );

        landmark.setMeshVicinity(new MeshVicinity(0, rayIntersection.getPosition()));

        return landmark;

    }

}

Detection/src/main/nbm/manifest.mf

deleted100644 → 0
+0 −3
Original line number Diff line number Diff line 
Manifest-Version: 1.0

OpenIDE-Module-Localizing-Bundle: cz/fidentis/detection/Bundle.properties

Detection/src/main/resources/cz/fidentis/detection/Bundle.properties

deleted100644 → 0
+0 −6
Original line number Diff line number Diff line 
#Localized module labels. Defaults taken from POM (<name>, <description>, <groupId>) if unset.

#OpenIDE-Module-Name=

#OpenIDE-Module-Short-Description=

#OpenIDE-Module-Long-Description=

#OpenIDE-Module-Display-Category=

#Fri Apr 26 10:57:07 CEST 2024

FaceData/src/main/java/cz/fidentis/analyst/data/face/HumanFace.java

+9 −1
Original line number Diff line number Diff line
@@ -40,7 +40,7 @@ public interface HumanFace extends HumanFaceEventBus, Serializable { 
    /**

     * Sets the symmetry plane. If the input argument is {@code null}, then removes the plane.

     *

     * @param plane The new symmetry plane; Must not be {@code null}

     * @param plane The new symmetry plane

     */

    void setSymmetryPlane(Plane plane);
@@ -119,6 +119,14 @@ public interface HumanFace extends HumanFaceEventBus, Serializable { 
     */

    String getShortName();



    /**

     * Returns canonical path to the folder in which the face file is located, i.e.,

     * {@link #getPath()} with {@link #getShortName()} suffix removed.

     *

     * @return Returns canonical path to the folder in which the face file is located

     */

    String getDirectory();



    /**

     * Returns already computed octree of the triangular mesh or {@code null}.

     * @return Already computed octree of the triangular mesh or {@code null}